{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Bert-fine-tune.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "8Odafj9TLwqC",
        "n62kMJN5NCA6",
        "dPTrpbTCMd9M",
        "A2RxGD-GY0kr",
        "InYLv2MQMpur",
        "LhLqzEyANJVh",
        "f9FfLiAkN0pl",
        "m4N7O1ysOh7k",
        "_9vogH6ede6i",
        "CxBVPcUBdo4W"
      ]
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "4ac518ea1437416c9b2a2da5d3633d18": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_ed9757fa4d4b4983b322ccb175072cef",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_a7c44dafe7364f1aac246a05ed885c27",
              "IPY_MODEL_af6d8df54f43461fa6f22460d300d06c"
            ]
          }
        },
        "ed9757fa4d4b4983b322ccb175072cef": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a7c44dafe7364f1aac246a05ed885c27": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_87be6d294ea042aea749ff73473e0247",
            "_dom_classes": [],
            "description": "Downloading",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 361,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 361,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_af074e64b3d14a1a98667a3db28f1787"
          }
        },
        "af6d8df54f43461fa6f22460d300d06c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_00ed2e1f0b22405da4234a3773933a24",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100% 361/361 [00:00&lt;00:00, 12.4kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_06a2b69d0b2a4a7dbdd7e3f82eea1aee"
          }
        },
        "87be6d294ea042aea749ff73473e0247": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "af074e64b3d14a1a98667a3db28f1787": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "00ed2e1f0b22405da4234a3773933a24": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "06a2b69d0b2a4a7dbdd7e3f82eea1aee": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "804e805509a646a78d3dcaf71e812439": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_1f040d45d3a54bc8bed4aa3294168568",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_3ffddf3586c447d88d445624d40908e4",
              "IPY_MODEL_1abae245e5e344c08e4fa1e110012230"
            ]
          }
        },
        "1f040d45d3a54bc8bed4aa3294168568": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3ffddf3586c447d88d445624d40908e4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_efd3e94d54ef4291b014c4242e01aa7e",
            "_dom_classes": [],
            "description": "Downloading",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 440473133,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 440473133,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_08b1426c483844c0a48ef4ef5482515a"
          }
        },
        "1abae245e5e344c08e4fa1e110012230": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_b3e8231be8f6462992b63acf3c52f2c7",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100% 440M/440M [00:05&lt;00:00, 76.4MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e35a024de6444bf288078bf5dc82358d"
          }
        },
        "efd3e94d54ef4291b014c4242e01aa7e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "08b1426c483844c0a48ef4ef5482515a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b3e8231be8f6462992b63acf3c52f2c7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e35a024de6444bf288078bf5dc82358d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "847dd893dd25496db43858fcbd8a1541": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_1b42036a91564750bb34824a7cb011dc",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_25894991c0934c6c87d5ccbe6e4960d2",
              "IPY_MODEL_4855e897b1c24414927c9d4ec34ec83d"
            ]
          }
        },
        "1b42036a91564750bb34824a7cb011dc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "25894991c0934c6c87d5ccbe6e4960d2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_ac53f834df6e4f10a9ff91a160facc0d",
            "_dom_classes": [],
            "description": "Downloading",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_aad6f87c10b04d1092808b205a586016"
          }
        },
        "4855e897b1c24414927c9d4ec34ec83d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_071e89c77d804d3dbee37f3db6ef3523",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100% 232k/232k [00:00&lt;00:00, 2.65MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d3692e8a7ed54fd8b68e470d377b596e"
          }
        },
        "ac53f834df6e4f10a9ff91a160facc0d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "aad6f87c10b04d1092808b205a586016": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "071e89c77d804d3dbee37f3db6ef3523": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d3692e8a7ed54fd8b68e470d377b596e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8Odafj9TLwqC",
        "colab_type": "text"
      },
      "source": [
        "# **Setup**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n62kMJN5NCA6",
        "colab_type": "text"
      },
      "source": [
        "### Imports and Installation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CoUupBnxw3a_",
        "colab_type": "code",
        "outputId": "35fde00b-1ddb-4e87-e2a8-cc71692fd5e5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 705
        }
      },
      "source": [
        "!pip install transformers\n",
        "\n",
        "import tensorflow as tf\n",
        "import torch\n",
        "import os\n",
        "import timeit\n",
        "from transformers.data.processors.squad import SquadResult, SquadV2Processor, squad_convert_examples_to_features\n",
        "from transformers import BertConfig, BertTokenizer, BertForQuestionAnswering, get_linear_schedule_with_warmup, AdamW\n",
        "from transformers.data.metrics.squad_metrics import compute_predictions_logits, squad_evaluate\n",
        "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n",
        "from tqdm import tqdm, trange"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ee/fc/bd726a15ab2c66dc09306689d04da07a3770dad724f0883f0a4bfb745087/transformers-2.4.1-py3-none-any.whl (475kB)\n",
            "\u001b[K     |████████████████████████████████| 481kB 3.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.17.5)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from transformers) (1.11.9)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.28.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
            "Collecting tokenizers==0.0.11\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/5e/36/7af38d572c935f8e0462ec7b4f7a46d73a2b3b1a938f50a5e8132d5b2dc5/tokenizers-0.0.11-cp36-cp36m-manylinux1_x86_64.whl (3.1MB)\n",
            "\u001b[K     |████████████████████████████████| 3.1MB 55.4MB/s \n",
            "\u001b[?25hCollecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a6/b4/7a41d630547a4afd58143597d5a49e07bfd4c42914d8335b2a5657efc14b/sacremoses-0.0.38.tar.gz (860kB)\n",
            "\u001b[K     |████████████████████████████████| 870kB 45.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.21.0)\n",
            "Collecting sentencepiece\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/74/f4/2d5214cbf13d06e7cb2c20d84115ca25b53ea76fa1f0ade0e3c9749de214/sentencepiece-0.1.85-cp36-cp36m-manylinux1_x86_64.whl (1.0MB)\n",
            "\u001b[K     |████████████████████████████████| 1.0MB 36.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: s3transfer<0.4.0,>=0.3.0 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (0.3.2)\n",
            "Requirement already satisfied: botocore<1.15.0,>=1.14.9 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (1.14.9)\n",
            "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (0.9.4)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.12.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.14.1)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2019.11.28)\n",
            "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.8)\n",
            "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.6/dist-packages (from botocore<1.15.0,>=1.14.9->boto3->transformers) (2.6.1)\n",
            "Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.15.0,>=1.14.9->boto3->transformers) (0.15.2)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.38-cp36-none-any.whl size=884628 sha256=7430ee3dc02a8c1ff8288770db77f6717f24e2a7fb04ed09d7e98c74dbe55475\n",
            "  Stored in directory: /root/.cache/pip/wheels/6d/ec/1a/21b8912e35e02741306f35f66c785f3afe94de754a0eaf1422\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: tokenizers, sacremoses, sentencepiece, transformers\n",
            "Successfully installed sacremoses-0.0.38 sentencepiece-0.1.85 tokenizers-0.0.11 transformers-2.4.1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p style=\"color: red;\">\n",
              "The default version of TensorFlow in Colab will soon switch to TensorFlow 2.x.<br>\n",
              "We recommend you <a href=\"https://www.tensorflow.org/guide/migrate\" target=\"_blank\">upgrade</a> now \n",
              "or ensure your notebook will continue to use TensorFlow 1.x via the <code>%tensorflow_version 1.x</code> magic:\n",
              "<a href=\"https://colab.research.google.com/notebooks/tensorflow_version.ipynb\" target=\"_blank\">more info</a>.</p>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bp4W-NqPYzq1",
        "colab_type": "text"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dPTrpbTCMd9M",
        "colab_type": "text"
      },
      "source": [
        "### GPU configuration"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TIC010r_6Pon",
        "colab_type": "code",
        "outputId": "96f35c28-ab7e-4d93-acb8-96d3ea527b8d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "# Get the GPU device name.\n",
        "device_name = tf.test.gpu_device_name()\n",
        "\n",
        "# The device name should look like the following:\n",
        "if device_name == '/device:GPU:0':\n",
        "    print('Found GPU at: {}'.format(device_name))\n",
        "else:\n",
        "    raise SystemError('GPU device not found')"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found GPU at: /device:GPU:0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zO2LSlJz7cbU",
        "colab_type": "code",
        "outputId": "89c487f4-7349-481b-c11a-cc33756af78d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "# If there's a GPU available...\n",
        "if torch.cuda.is_available():    \n",
        "\n",
        "    # Tell PyTorch to use the GPU.    \n",
        "    device = torch.device(\"cuda\")\n",
        "\n",
        "    print('There are %d GPU(s) available.' % torch.cuda.device_count())\n",
        "\n",
        "    print('We will use the GPU:', torch.cuda.get_device_name(0))\n",
        "\n",
        "# If not...\n",
        "else:\n",
        "    print('No GPU available, using the CPU instead.')\n",
        "    device = torch.device(\"cpu\")"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "There are 1 GPU(s) available.\n",
            "We will use the GPU: Tesla T4\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A2RxGD-GY0kr",
        "colab_type": "text"
      },
      "source": [
        "# **Loading & Processing the SQuAD 2.0 Dataset**\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "InYLv2MQMpur",
        "colab_type": "text"
      },
      "source": [
        "## Downloading Squad files :"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5L4i7OBdD0Vr",
        "colab_type": "code",
        "outputId": "56fd77f5-94dd-4dca-cd25-d3bbb88c6fa2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 523
        }
      },
      "source": [
        "#Download the SQUAD train and dev dataset\n",
        "!mkdir squaddir\n",
        "!wget https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v2.0.json -O squaddir/train-v2.0.json\n",
        "!wget https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v2.0.json -O squaddir/dev-v2.0.json\n",
        "!wget https://worksheets.codalab.org/rest/bundles/0x6b567e1cf2e041ec80d7098f031c5c9e/contents/blob/ -O squaddir/evaluate-v2.0.py"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-02-06 16:26:39--  https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v2.0.json\n",
            "Resolving rajpurkar.github.io (rajpurkar.github.io)... 185.199.108.153, 185.199.110.153, 185.199.111.153, ...\n",
            "Connecting to rajpurkar.github.io (rajpurkar.github.io)|185.199.108.153|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 42123633 (40M) [application/json]\n",
            "Saving to: ‘squaddir/train-v2.0.json’\n",
            "\n",
            "squaddir/train-v2.0 100%[===================>]  40.17M  70.6MB/s    in 0.6s    \n",
            "\n",
            "2020-02-06 16:26:40 (70.6 MB/s) - ‘squaddir/train-v2.0.json’ saved [42123633/42123633]\n",
            "\n",
            "--2020-02-06 16:26:41--  https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v2.0.json\n",
            "Resolving rajpurkar.github.io (rajpurkar.github.io)... 185.199.108.153, 185.199.110.153, 185.199.111.153, ...\n",
            "Connecting to rajpurkar.github.io (rajpurkar.github.io)|185.199.108.153|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 4370528 (4.2M) [application/json]\n",
            "Saving to: ‘squaddir/dev-v2.0.json’\n",
            "\n",
            "squaddir/dev-v2.0.j 100%[===================>]   4.17M  15.0MB/s    in 0.3s    \n",
            "\n",
            "2020-02-06 16:26:41 (15.0 MB/s) - ‘squaddir/dev-v2.0.json’ saved [4370528/4370528]\n",
            "\n",
            "--2020-02-06 16:26:42--  https://worksheets.codalab.org/rest/bundles/0x6b567e1cf2e041ec80d7098f031c5c9e/contents/blob/\n",
            "Resolving worksheets.codalab.org (worksheets.codalab.org)... 40.71.231.153\n",
            "Connecting to worksheets.codalab.org (worksheets.codalab.org)|40.71.231.153|:443... connected.\n",
            "ERROR: cannot verify worksheets.codalab.org's certificate, issued by ‘CN=Go Daddy Secure Certificate Authority - G2,OU=http://certs.godaddy.com/repository/,O=GoDaddy.com\\\\, Inc.,L=Scottsdale,ST=Arizona,C=US’:\n",
            "  Issued certificate has expired.\n",
            "To connect to worksheets.codalab.org insecurely, use `--no-check-certificate'.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LhLqzEyANJVh",
        "colab_type": "text"
      },
      "source": [
        "## BERT Model Loading"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w7k-PkW1XHTQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# PARAMETERS\n",
        "\n",
        "pretrained_weights = 'bert-base-uncased'\n",
        "max_seq_length = 384\n",
        "max_answer_length = 30\n",
        "max_query_length = 64\n",
        "doc_stride = 128\n",
        "is_training = True\n",
        "return_dataset = \"pt\"\n",
        "learning_rate = 3e-5\n",
        "num_train_epochs = 4\n",
        "output_dir = \"./finetuned_squad/\"\n",
        "eval_batch_size=10\n",
        "train_batch_size=10\n",
        "gradient_accumulation_steps = 1\n",
        "weight_decay = 0.0\n",
        "adam_epsilon = 1e-8\n",
        "warmup_steps = 0\n",
        "logging_steps = 500\n",
        "max_grad_norm = 1.0\n",
        "verbose_logging = True\n",
        "version_2_with_negative = True\n",
        "null_score_diff_threshold = 0.0\n",
        "do_lower_case = True\n",
        "n_best_size = 20"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i1fsCm8GNgHO",
        "colab_type": "code",
        "outputId": "988ebb59-72e4-41ec-99ff-b24c6fc3a02f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 217,
          "referenced_widgets": [
            "4ac518ea1437416c9b2a2da5d3633d18",
            "ed9757fa4d4b4983b322ccb175072cef",
            "a7c44dafe7364f1aac246a05ed885c27",
            "af6d8df54f43461fa6f22460d300d06c",
            "87be6d294ea042aea749ff73473e0247",
            "af074e64b3d14a1a98667a3db28f1787",
            "00ed2e1f0b22405da4234a3773933a24",
            "06a2b69d0b2a4a7dbdd7e3f82eea1aee",
            "804e805509a646a78d3dcaf71e812439",
            "1f040d45d3a54bc8bed4aa3294168568",
            "3ffddf3586c447d88d445624d40908e4",
            "1abae245e5e344c08e4fa1e110012230",
            "efd3e94d54ef4291b014c4242e01aa7e",
            "08b1426c483844c0a48ef4ef5482515a",
            "b3e8231be8f6462992b63acf3c52f2c7",
            "e35a024de6444bf288078bf5dc82358d",
            "847dd893dd25496db43858fcbd8a1541",
            "1b42036a91564750bb34824a7cb011dc",
            "25894991c0934c6c87d5ccbe6e4960d2",
            "4855e897b1c24414927c9d4ec34ec83d",
            "ac53f834df6e4f10a9ff91a160facc0d",
            "aad6f87c10b04d1092808b205a586016",
            "071e89c77d804d3dbee37f3db6ef3523",
            "d3692e8a7ed54fd8b68e470d377b596e"
          ]
        }
      },
      "source": [
        "# Loading the model config\n",
        "print(\"Bert config loading ...\")\n",
        "config = BertConfig.from_pretrained(pretrained_weights)\n",
        "\n",
        "# Loading the model\n",
        "print(\"BertForQuestionAnswering model loading ...\")\n",
        "model = BertForQuestionAnswering.from_pretrained(pretrained_weights)\n",
        "model.cuda()\n",
        "\n",
        "# Loading the tokenizer\n",
        "print(\"BertTokenizer loading ...\")\n",
        "tokenizer = BertTokenizer.from_pretrained(pretrained_weights)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Bert config loading ...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4ac518ea1437416c9b2a2da5d3633d18",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, description='Downloading', max=361, style=ProgressStyle(description_width=…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "BertForQuestionAnswering model loading ...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "804e805509a646a78d3dcaf71e812439",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, description='Downloading', max=440473133, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "BertTokenizer loading ...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "847dd893dd25496db43858fcbd8a1541",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, description='Downloading', max=231508, style=ProgressStyle(description_wid…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1kkgshQsSYAo",
        "colab_type": "code",
        "outputId": "885bdc27-be35-4941-9799-e9bc52271097",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 624
        }
      },
      "source": [
        "# Get all of the model's parameters as a list of tuples.\n",
        "params = list(model.named_parameters())\n",
        "\n",
        "print('The BERT model has {:} different named parameters.\\n'.format(len(params)))\n",
        "\n",
        "print('==== Embedding Layer ====\\n')\n",
        "\n",
        "for p in params[0:5]:\n",
        "    print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))\n",
        "\n",
        "print('\\n==== First Transformer ====\\n')\n",
        "\n",
        "for p in params[5:21]:\n",
        "    print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))\n",
        "\n",
        "print('\\n==== Output Layer ====\\n')\n",
        "\n",
        "for p in params[-4:]:\n",
        "    print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The BERT model has 201 different named parameters.\n",
            "\n",
            "==== Embedding Layer ====\n",
            "\n",
            "bert.embeddings.word_embeddings.weight                  (30522, 768)\n",
            "bert.embeddings.position_embeddings.weight                (512, 768)\n",
            "bert.embeddings.token_type_embeddings.weight                (2, 768)\n",
            "bert.embeddings.LayerNorm.weight                              (768,)\n",
            "bert.embeddings.LayerNorm.bias                                (768,)\n",
            "\n",
            "==== First Transformer ====\n",
            "\n",
            "bert.encoder.layer.0.attention.self.query.weight          (768, 768)\n",
            "bert.encoder.layer.0.attention.self.query.bias                (768,)\n",
            "bert.encoder.layer.0.attention.self.key.weight            (768, 768)\n",
            "bert.encoder.layer.0.attention.self.key.bias                  (768,)\n",
            "bert.encoder.layer.0.attention.self.value.weight          (768, 768)\n",
            "bert.encoder.layer.0.attention.self.value.bias                (768,)\n",
            "bert.encoder.layer.0.attention.output.dense.weight        (768, 768)\n",
            "bert.encoder.layer.0.attention.output.dense.bias              (768,)\n",
            "bert.encoder.layer.0.attention.output.LayerNorm.weight        (768,)\n",
            "bert.encoder.layer.0.attention.output.LayerNorm.bias          (768,)\n",
            "bert.encoder.layer.0.intermediate.dense.weight           (3072, 768)\n",
            "bert.encoder.layer.0.intermediate.dense.bias                 (3072,)\n",
            "bert.encoder.layer.0.output.dense.weight                 (768, 3072)\n",
            "bert.encoder.layer.0.output.dense.bias                        (768,)\n",
            "bert.encoder.layer.0.output.LayerNorm.weight                  (768,)\n",
            "bert.encoder.layer.0.output.LayerNorm.bias                    (768,)\n",
            "\n",
            "==== Output Layer ====\n",
            "\n",
            "bert.pooler.dense.weight                                  (768, 768)\n",
            "bert.pooler.dense.bias                                        (768,)\n",
            "qa_outputs.weight                                           (2, 768)\n",
            "qa_outputs.bias                                                 (2,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "f9FfLiAkN0pl",
        "colab_type": "text"
      },
      "source": [
        "## Data processing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JOOBqTTkdRbs",
        "colab_type": "code",
        "outputId": "4a2e7c9c-a7c4-46bb-8037-0e459c965923",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 208
        }
      },
      "source": [
        "##############################\n",
        "## TRAIN DATASET PROCESSING ##\n",
        "##############################\n",
        "\n",
        "# Squad V2 processor loading\n",
        "print(\"\\nSquad V2 processor loading : \")\n",
        "processor = SquadV2Processor()\n",
        "\n",
        "# Extract examples from the train dataset\n",
        "print(\"\\nExtract examples from the train dataset : \")\n",
        "examples = processor.get_train_examples(\"squaddir\")\n",
        "\n",
        "print(\"\\nNumber of train examples total: \", len(examples))\n",
        "\n",
        "examples = examples[:10000]\n",
        "print(\"\\nNumber of train examples taken: \", len(examples))\n",
        "\n",
        "features, dataset = squad_convert_examples_to_features(\n",
        "    examples=examples,\n",
        "    tokenizer=tokenizer,\n",
        "    max_seq_length=max_seq_length,\n",
        "    doc_stride=doc_stride,\n",
        "    max_query_length=max_query_length,\n",
        "    is_training=is_training,\n",
        "    return_dataset=return_dataset\n",
        ")"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Squad V2 processor loading : \n",
            "\n",
            "Extract examples from the train dataset : \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 442/442 [00:40<00:00, 10.87it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Number of train examples total:  130319\n",
            "\n",
            "Number of train examples taken:  10000\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "convert squad examples to features: 100%|██████████| 10000/10000 [01:04<00:00, 156.02it/s]\n",
            "add example index and unique id: 100%|██████████| 10000/10000 [00:00<00:00, 831807.08it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eBGiCdHwOCuY",
        "colab_type": "code",
        "outputId": "974d7a2d-577d-4e7f-d20c-b7c17882956b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        }
      },
      "source": [
        "#############################\n",
        "## TEST DATASET PROCESSING ##\n",
        "#############################\n",
        "\n",
        "# Extract examples from the test dataset\n",
        "print(\"\\nExtract examples from the test dataset : \")\n",
        "dev_examples = processor.get_dev_examples(\"squaddir\")\n",
        "\n",
        "print(\"\\nNumber of test examples total: \", len(dev_examples))\n",
        "\n",
        "# dev_examples = dev_examples[:1000]\n",
        "# print(\"Number of test examples taken: \", len(dev_examples))\n",
        "\n",
        "dev_features, dev_dataset = squad_convert_examples_to_features(\n",
        "    examples=dev_examples,\n",
        "    tokenizer=tokenizer,\n",
        "    max_seq_length=max_seq_length,\n",
        "    doc_stride=doc_stride,\n",
        "    max_query_length=max_query_length,\n",
        "    is_training=not is_training,\n",
        "    return_dataset=return_dataset\n",
        ")"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Extract examples from the test dataset : \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 35/35 [00:03<00:00,  7.65it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Number of test examples total:  11873\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "convert squad examples to features: 100%|██████████| 11873/11873 [01:31<00:00, 129.63it/s]\n",
            "add example index and unique id: 100%|██████████| 11873/11873 [00:00<00:00, 763120.76it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UAmIzUZTCALh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# print(\"dataset 0 : \", dataset[0])\n",
        "\n",
        "# feat = features[0]\n",
        "\n",
        "# print(\"features 0 : \", feat)\n",
        "# print(\"input_ids : \", feat.input_ids)\n",
        "# print(\"attention_mask : \", feat.attention_mask)\n",
        "# print(\"token_type_ids : \", feat.token_type_ids)\n",
        "# print(\"cls_index : \", feat.cls_index)\n",
        "# print(\"p_mask : \", feat.p_mask)\n",
        "# print(\"example_index : \", feat.example_index)\n",
        "# print(\"unique_id : \", feat.unique_id)\n",
        "# print(\"paragraph_len : \", feat.paragraph_len)\n",
        "# print(\"token_is_max_context : \", feat.token_is_max_context)\n",
        "# print(\"tokens : \", feat.tokens)\n",
        "# print(\"token_to_orig_map : \", feat.token_to_orig_map)\n",
        "# print(\"start_position : \", feat.start_position)\n",
        "# print(\"end_position : \", feat.end_position)\n",
        "# print(\"is_impossible : \", feat.is_impossible)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "m4N7O1ysOh7k",
        "colab_type": "text"
      },
      "source": [
        "# **Training the model**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kfr5gVYNAhqp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def to_list(tensor):\n",
        "    return tensor.detach().cpu().tolist()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KcOBgJUfXe-c",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "loss_tab = []\n",
        "\n",
        "def train(train_dataset, model, tokenizer):\n",
        "    \"\"\" Train the model \"\"\"\n",
        "\n",
        "    train_sampler = RandomSampler(train_dataset)\n",
        "    train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=train_batch_size)\n",
        "\n",
        "    t_total = len(train_dataloader) // gradient_accumulation_steps * num_train_epochs\n",
        "\n",
        "    # Prepare optimizer and schedule (linear warmup and decay)\n",
        "    no_decay = [\"bias\", \"LayerNorm.weight\"]\n",
        "\n",
        "    optimizer_grouped_parameters = [\n",
        "        {\n",
        "            \"params\": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],\n",
        "            \"weight_decay\": weight_decay,\n",
        "        },\n",
        "        {\"params\": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], \"weight_decay\": 0.0},\n",
        "    ]\n",
        "\n",
        "    optimizer = AdamW(optimizer_grouped_parameters, lr=learning_rate, eps=adam_epsilon)\n",
        "\n",
        "    scheduler = get_linear_schedule_with_warmup(\n",
        "        optimizer, num_warmup_steps=warmup_steps, num_training_steps=t_total\n",
        "    )\n",
        "\n",
        "    # Train!\n",
        "    print(\"***** Running training *****\")\n",
        "    print(\"  Num examples = \", len(train_dataset))\n",
        "    print(\"  Num Epochs = \", num_train_epochs)\n",
        "    print(\"  Gradient Accumulation steps = \", gradient_accumulation_steps)\n",
        "    print(\"  Total optimization steps = \", t_total)\n",
        "\n",
        "    global_step = 1\n",
        "    epochs_trained = 0\n",
        "    steps_trained_in_current_epoch = 0\n",
        "\n",
        "    tr_loss, logging_loss = 0.0, 0.0\n",
        "    model.zero_grad()\n",
        "\n",
        "    train_iterator = trange(\n",
        "        epochs_trained, int(num_train_epochs), desc=\"Epoch\"\n",
        "    )\n",
        "\n",
        "    for _ in train_iterator:\n",
        "        epoch_iterator = tqdm(train_dataloader, desc=\"Iteration\")\n",
        "        for step, batch in enumerate(epoch_iterator):\n",
        "\n",
        "            # Skip past any already trained steps if resuming training\n",
        "            if steps_trained_in_current_epoch > 0:\n",
        "                steps_trained_in_current_epoch -= 1\n",
        "                continue\n",
        "\n",
        "            model.train()\n",
        "            batch = tuple(t.to(device) for t in batch)\n",
        "\n",
        "            inputs = {\n",
        "                \"input_ids\": batch[0],\n",
        "                \"attention_mask\": batch[1],\n",
        "                \"token_type_ids\": batch[2],\n",
        "                \"start_positions\": batch[3],\n",
        "                \"end_positions\": batch[4],\n",
        "            }\n",
        "\n",
        "            outputs = model(**inputs)\n",
        "            # model outputs are always tuple in transformers (see doc)\n",
        "            loss = outputs[0]\n",
        "\n",
        "            loss.backward()\n",
        "\n",
        "            tr_loss += loss.item()\n",
        "\n",
        "            torch.nn.utils.clip_grad_norm_(model.parameters(), max_grad_norm)\n",
        "\n",
        "            optimizer.step()\n",
        "            scheduler.step()  # Update learning rate schedule\n",
        "            model.zero_grad()\n",
        "            global_step += 1\n",
        "        \n",
        "        loss_tab.append(tr_loss / global_step)\n",
        "\n",
        "    return global_step, tr_loss / global_step"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "blYof7D3gn6s",
        "colab_type": "code",
        "outputId": "bde44ad9-aec3-4013-c6d8-f942e464e5a3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "global_step, tr_loss = train(dataset, model, tokenizer)\n",
        "print(\" global_step = \", global_step, \", average loss = \",  tr_loss)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch:   0%|          | 0/4 [00:00<?, ?it/s]\n",
            "Iteration:   0%|          | 0/1015 [00:00<?, ?it/s]\u001b[A"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "***** Running training *****\n",
            "  Num examples =  10150\n",
            "  Num Epochs =  4\n",
            "  Gradient Accumulation steps =  1\n",
            "  Total optimization steps =  4060\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Iteration:   0%|          | 1/1015 [00:01<17:53,  1.06s/it]\u001b[A\n",
            "Iteration:   0%|          | 2/1015 [00:01<16:40,  1.01it/s]\u001b[A\n",
            "Iteration:   0%|          | 3/1015 [00:02<15:46,  1.07it/s]\u001b[A\n",
            "Iteration:   0%|          | 4/1015 [00:03<15:14,  1.11it/s]\u001b[A\n",
            "Iteration:   0%|          | 5/1015 [00:04<14:50,  1.13it/s]\u001b[A\n",
            "Iteration:   1%|          | 6/1015 [00:05<14:34,  1.15it/s]\u001b[A\n",
            "Iteration:   1%|          | 7/1015 [00:06<14:22,  1.17it/s]\u001b[A\n",
            "Iteration:   1%|          | 8/1015 [00:06<14:14,  1.18it/s]\u001b[A\n",
            "Iteration:   1%|          | 9/1015 [00:07<14:08,  1.19it/s]\u001b[A\n",
            "Iteration:   1%|          | 10/1015 [00:08<14:04,  1.19it/s]\u001b[A\n",
            "Iteration:   1%|          | 11/1015 [00:09<14:05,  1.19it/s]\u001b[A\n",
            "Iteration:   1%|          | 12/1015 [00:10<14:04,  1.19it/s]\u001b[A\n",
            "Iteration:   1%|▏         | 13/1015 [00:11<14:02,  1.19it/s]\u001b[A\n",
            "Iteration:   1%|▏         | 14/1015 [00:11<13:58,  1.19it/s]\u001b[A\n",
            "Iteration:   1%|▏         | 15/1015 [00:12<13:57,  1.19it/s]\u001b[A\n",
            "Iteration:   2%|▏         | 16/1015 [00:13<13:53,  1.20it/s]\u001b[A\n",
            "Iteration:   2%|▏         | 17/1015 [00:14<13:51,  1.20it/s]\u001b[A\n",
            "Iteration:   2%|▏         | 18/1015 [00:15<13:54,  1.19it/s]\u001b[A\n",
            "Iteration:   2%|▏         | 19/1015 [00:16<13:53,  1.20it/s]\u001b[A\n",
            "Iteration:   2%|▏         | 20/1015 [00:16<13:57,  1.19it/s]\u001b[A\n",
            "Iteration:   2%|▏         | 21/1015 [00:17<13:57,  1.19it/s]\u001b[A\n",
            "Iteration:   2%|▏         | 22/1015 [00:18<13:54,  1.19it/s]\u001b[A\n",
            "Iteration:   2%|▏         | 23/1015 [00:19<13:55,  1.19it/s]\u001b[A\n",
            "Iteration:   2%|▏         | 24/1015 [00:20<13:57,  1.18it/s]\u001b[A\n",
            "Iteration:   2%|▏         | 25/1015 [00:21<13:59,  1.18it/s]\u001b[A\n",
            "Iteration:   3%|▎         | 26/1015 [00:21<13:59,  1.18it/s]\u001b[A\n",
            "Iteration:   3%|▎         | 27/1015 [00:22<14:01,  1.17it/s]\u001b[A\n",
            "Iteration:   3%|▎         | 28/1015 [00:23<14:00,  1.17it/s]\u001b[A\n",
            "Iteration:   3%|▎         | 29/1015 [00:24<14:01,  1.17it/s]\u001b[A\n",
            "Iteration:   3%|▎         | 30/1015 [00:25<14:04,  1.17it/s]\u001b[A\n",
            "Iteration:   3%|▎         | 31/1015 [00:26<14:03,  1.17it/s]\u001b[A\n",
            "Iteration:   3%|▎         | 32/1015 [00:27<14:06,  1.16it/s]\u001b[A\n",
            "Iteration:   3%|▎         | 33/1015 [00:28<14:06,  1.16it/s]\u001b[A\n",
            "Iteration:   3%|▎         | 34/1015 [00:28<14:11,  1.15it/s]\u001b[A\n",
            "Iteration:   3%|▎         | 35/1015 [00:29<14:13,  1.15it/s]\u001b[A\n",
            "Iteration:   4%|▎         | 36/1015 [00:30<14:14,  1.15it/s]\u001b[A\n",
            "Iteration:   4%|▎         | 37/1015 [00:31<14:15,  1.14it/s]\u001b[A\n",
            "Iteration:   4%|▎         | 38/1015 [00:32<14:15,  1.14it/s]\u001b[A\n",
            "Iteration:   4%|▍         | 39/1015 [00:33<14:17,  1.14it/s]\u001b[A\n",
            "Iteration:   4%|▍         | 40/1015 [00:34<14:17,  1.14it/s]\u001b[A\n",
            "Iteration:   4%|▍         | 41/1015 [00:35<14:19,  1.13it/s]\u001b[A\n",
            "Iteration:   4%|▍         | 42/1015 [00:35<14:21,  1.13it/s]\u001b[A\n",
            "Iteration:   4%|▍         | 43/1015 [00:36<14:23,  1.13it/s]\u001b[A\n",
            "Iteration:   4%|▍         | 44/1015 [00:37<14:26,  1.12it/s]\u001b[A\n",
            "Iteration:   4%|▍         | 45/1015 [00:38<14:27,  1.12it/s]\u001b[A\n",
            "Iteration:   5%|▍         | 46/1015 [00:39<14:28,  1.12it/s]\u001b[A\n",
            "Iteration:   5%|▍         | 47/1015 [00:40<14:28,  1.11it/s]\u001b[A\n",
            "Iteration:   5%|▍         | 48/1015 [00:41<14:28,  1.11it/s]\u001b[A\n",
            "Iteration:   5%|▍         | 49/1015 [00:42<14:31,  1.11it/s]\u001b[A\n",
            "Iteration:   5%|▍         | 50/1015 [00:43<14:32,  1.11it/s]\u001b[A\n",
            "Iteration:   5%|▌         | 51/1015 [00:44<14:33,  1.10it/s]\u001b[A\n",
            "Iteration:   5%|▌         | 52/1015 [00:44<14:33,  1.10it/s]\u001b[A\n",
            "Iteration:   5%|▌         | 53/1015 [00:45<14:37,  1.10it/s]\u001b[A\n",
            "Iteration:   5%|▌         | 54/1015 [00:46<14:40,  1.09it/s]\u001b[A\n",
            "Iteration:   5%|▌         | 55/1015 [00:47<14:42,  1.09it/s]\u001b[A\n",
            "Iteration:   6%|▌         | 56/1015 [00:48<14:41,  1.09it/s]\u001b[A\n",
            "Iteration:   6%|▌         | 57/1015 [00:49<14:43,  1.08it/s]\u001b[A\n",
            "Iteration:   6%|▌         | 58/1015 [00:50<14:43,  1.08it/s]\u001b[A\n",
            "Iteration:   6%|▌         | 59/1015 [00:51<14:45,  1.08it/s]\u001b[A\n",
            "Iteration:   6%|▌         | 60/1015 [00:52<14:44,  1.08it/s]\u001b[A\n",
            "Iteration:   6%|▌         | 61/1015 [00:53<14:43,  1.08it/s]\u001b[A\n",
            "Iteration:   6%|▌         | 62/1015 [00:54<14:44,  1.08it/s]\u001b[A\n",
            "Iteration:   6%|▌         | 63/1015 [00:55<14:42,  1.08it/s]\u001b[A\n",
            "Iteration:   6%|▋         | 64/1015 [00:56<14:43,  1.08it/s]\u001b[A\n",
            "Iteration:   6%|▋         | 65/1015 [00:57<14:42,  1.08it/s]\u001b[A\n",
            "Iteration:   7%|▋         | 66/1015 [00:57<14:39,  1.08it/s]\u001b[A\n",
            "Iteration:   7%|▋         | 67/1015 [00:58<14:38,  1.08it/s]\u001b[A\n",
            "Iteration:   7%|▋         | 68/1015 [00:59<14:35,  1.08it/s]\u001b[A\n",
            "Iteration:   7%|▋         | 69/1015 [01:00<14:33,  1.08it/s]\u001b[A\n",
            "Iteration:   7%|▋         | 70/1015 [01:01<14:28,  1.09it/s]\u001b[A\n",
            "Iteration:   7%|▋         | 71/1015 [01:02<14:26,  1.09it/s]\u001b[A\n",
            "Iteration:   7%|▋         | 72/1015 [01:03<14:22,  1.09it/s]\u001b[A\n",
            "Iteration:   7%|▋         | 73/1015 [01:04<14:17,  1.10it/s]\u001b[A\n",
            "Iteration:   7%|▋         | 74/1015 [01:05<14:17,  1.10it/s]\u001b[A\n",
            "Iteration:   7%|▋         | 75/1015 [01:06<14:15,  1.10it/s]\u001b[A\n",
            "Iteration:   7%|▋         | 76/1015 [01:07<14:12,  1.10it/s]\u001b[A\n",
            "Iteration:   8%|▊         | 77/1015 [01:07<14:09,  1.10it/s]\u001b[A\n",
            "Iteration:   8%|▊         | 78/1015 [01:08<14:07,  1.11it/s]\u001b[A\n",
            "Iteration:   8%|▊         | 79/1015 [01:09<14:05,  1.11it/s]\u001b[A\n",
            "Iteration:   8%|▊         | 80/1015 [01:10<14:01,  1.11it/s]\u001b[A\n",
            "Iteration:   8%|▊         | 81/1015 [01:11<13:59,  1.11it/s]\u001b[A\n",
            "Iteration:   8%|▊         | 82/1015 [01:12<13:59,  1.11it/s]\u001b[A\n",
            "Iteration:   8%|▊         | 83/1015 [01:13<13:56,  1.11it/s]\u001b[A\n",
            "Iteration:   8%|▊         | 84/1015 [01:14<13:53,  1.12it/s]\u001b[A\n",
            "Iteration:   8%|▊         | 85/1015 [01:15<13:50,  1.12it/s]\u001b[A\n",
            "Iteration:   8%|▊         | 86/1015 [01:16<13:48,  1.12it/s]\u001b[A\n",
            "Iteration:   9%|▊         | 87/1015 [01:16<13:46,  1.12it/s]\u001b[A\n",
            "Iteration:   9%|▊         | 88/1015 [01:17<13:43,  1.13it/s]\u001b[A\n",
            "Iteration:   9%|▉         | 89/1015 [01:18<13:42,  1.13it/s]\u001b[A\n",
            "Iteration:   9%|▉         | 90/1015 [01:19<13:41,  1.13it/s]\u001b[A\n",
            "Iteration:   9%|▉         | 91/1015 [01:20<13:38,  1.13it/s]\u001b[A\n",
            "Iteration:   9%|▉         | 92/1015 [01:21<13:36,  1.13it/s]\u001b[A\n",
            "Iteration:   9%|▉         | 93/1015 [01:22<13:38,  1.13it/s]\u001b[A\n",
            "Iteration:   9%|▉         | 94/1015 [01:23<13:35,  1.13it/s]\u001b[A\n",
            "Iteration:   9%|▉         | 95/1015 [01:23<13:31,  1.13it/s]\u001b[A\n",
            "Iteration:   9%|▉         | 96/1015 [01:24<13:27,  1.14it/s]\u001b[A\n",
            "Iteration:  10%|▉         | 97/1015 [01:25<13:25,  1.14it/s]\u001b[A\n",
            "Iteration:  10%|▉         | 98/1015 [01:26<13:25,  1.14it/s]\u001b[A\n",
            "Iteration:  10%|▉         | 99/1015 [01:27<13:23,  1.14it/s]\u001b[A\n",
            "Iteration:  10%|▉         | 100/1015 [01:28<13:20,  1.14it/s]\u001b[A\n",
            "Iteration:  10%|▉         | 101/1015 [01:29<13:18,  1.14it/s]\u001b[A\n",
            "Iteration:  10%|█         | 102/1015 [01:30<13:16,  1.15it/s]\u001b[A\n",
            "Iteration:  10%|█         | 103/1015 [01:30<13:19,  1.14it/s]\u001b[A\n",
            "Iteration:  10%|█         | 104/1015 [01:31<13:17,  1.14it/s]\u001b[A\n",
            "Iteration:  10%|█         | 105/1015 [01:32<13:15,  1.14it/s]\u001b[A\n",
            "Iteration:  10%|█         | 106/1015 [01:33<13:12,  1.15it/s]\u001b[A\n",
            "Iteration:  11%|█         | 107/1015 [01:34<13:12,  1.15it/s]\u001b[A\n",
            "Iteration:  11%|█         | 108/1015 [01:35<13:12,  1.14it/s]\u001b[A\n",
            "Iteration:  11%|█         | 109/1015 [01:36<13:13,  1.14it/s]\u001b[A\n",
            "Iteration:  11%|█         | 110/1015 [01:37<13:14,  1.14it/s]\u001b[A\n",
            "Iteration:  11%|█         | 111/1015 [01:37<13:13,  1.14it/s]\u001b[A\n",
            "Iteration:  11%|█         | 112/1015 [01:38<13:10,  1.14it/s]\u001b[A\n",
            "Iteration:  11%|█         | 113/1015 [01:39<13:08,  1.14it/s]\u001b[A\n",
            "Iteration:  11%|█         | 114/1015 [01:40<13:06,  1.15it/s]\u001b[A\n",
            "Iteration:  11%|█▏        | 115/1015 [01:41<13:04,  1.15it/s]\u001b[A\n",
            "Iteration:  11%|█▏        | 116/1015 [01:42<13:04,  1.15it/s]\u001b[A\n",
            "Iteration:  12%|█▏        | 117/1015 [01:43<13:02,  1.15it/s]\u001b[A\n",
            "Iteration:  12%|█▏        | 118/1015 [01:44<13:02,  1.15it/s]\u001b[A\n",
            "Iteration:  12%|█▏        | 119/1015 [01:44<12:59,  1.15it/s]\u001b[A\n",
            "Iteration:  12%|█▏        | 120/1015 [01:45<12:58,  1.15it/s]\u001b[A\n",
            "Iteration:  12%|█▏        | 121/1015 [01:46<12:59,  1.15it/s]\u001b[A\n",
            "Iteration:  12%|█▏        | 122/1015 [01:47<12:58,  1.15it/s]\u001b[A\n",
            "Iteration:  12%|█▏        | 123/1015 [01:48<13:00,  1.14it/s]\u001b[A\n",
            "Iteration:  12%|█▏        | 124/1015 [01:49<12:59,  1.14it/s]\u001b[A\n",
            "Iteration:  12%|█▏        | 125/1015 [01:50<12:57,  1.14it/s]\u001b[A\n",
            "Iteration:  12%|█▏        | 126/1015 [01:51<12:58,  1.14it/s]\u001b[A\n",
            "Iteration:  13%|█▎        | 127/1015 [01:51<12:57,  1.14it/s]\u001b[A\n",
            "Iteration:  13%|█▎        | 128/1015 [01:52<12:56,  1.14it/s]\u001b[A\n",
            "Iteration:  13%|█▎        | 129/1015 [01:53<12:54,  1.14it/s]\u001b[A\n",
            "Iteration:  13%|█▎        | 130/1015 [01:54<12:56,  1.14it/s]\u001b[A\n",
            "Iteration:  13%|█▎        | 131/1015 [01:55<12:53,  1.14it/s]\u001b[A\n",
            "Iteration:  13%|█▎        | 132/1015 [01:56<12:55,  1.14it/s]\u001b[A\n",
            "Iteration:  13%|█▎        | 133/1015 [01:57<12:54,  1.14it/s]\u001b[A\n",
            "Iteration:  13%|█▎        | 134/1015 [01:58<12:52,  1.14it/s]\u001b[A\n",
            "Iteration:  13%|█▎        | 135/1015 [01:58<12:52,  1.14it/s]\u001b[A\n",
            "Iteration:  13%|█▎        | 136/1015 [01:59<12:51,  1.14it/s]\u001b[A\n",
            "Iteration:  13%|█▎        | 137/1015 [02:00<12:50,  1.14it/s]\u001b[A\n",
            "Iteration:  14%|█▎        | 138/1015 [02:01<12:50,  1.14it/s]\u001b[A\n",
            "Iteration:  14%|█▎        | 139/1015 [02:02<12:50,  1.14it/s]\u001b[A\n",
            "Iteration:  14%|█▍        | 140/1015 [02:03<12:49,  1.14it/s]\u001b[A\n",
            "Iteration:  14%|█▍        | 141/1015 [02:04<12:50,  1.13it/s]\u001b[A\n",
            "Iteration:  14%|█▍        | 142/1015 [02:05<12:49,  1.14it/s]\u001b[A\n",
            "Iteration:  14%|█▍        | 143/1015 [02:06<12:48,  1.13it/s]\u001b[A\n",
            "Iteration:  14%|█▍        | 144/1015 [02:06<12:49,  1.13it/s]\u001b[A\n",
            "Iteration:  14%|█▍        | 145/1015 [02:07<12:47,  1.13it/s]\u001b[A\n",
            "Iteration:  14%|█▍        | 146/1015 [02:08<12:48,  1.13it/s]\u001b[A\n",
            "Iteration:  14%|█▍        | 147/1015 [02:09<12:48,  1.13it/s]\u001b[A\n",
            "Iteration:  15%|█▍        | 148/1015 [02:10<12:48,  1.13it/s]\u001b[A\n",
            "Iteration:  15%|█▍        | 149/1015 [02:11<12:47,  1.13it/s]\u001b[A\n",
            "Iteration:  15%|█▍        | 150/1015 [02:12<12:46,  1.13it/s]\u001b[A\n",
            "Iteration:  15%|█▍        | 151/1015 [02:13<12:45,  1.13it/s]\u001b[A\n",
            "Iteration:  15%|█▍        | 152/1015 [02:13<12:45,  1.13it/s]\u001b[A\n",
            "Iteration:  15%|█▌        | 153/1015 [02:14<12:44,  1.13it/s]\u001b[A\n",
            "Iteration:  15%|█▌        | 154/1015 [02:15<12:43,  1.13it/s]\u001b[A\n",
            "Iteration:  15%|█▌        | 155/1015 [02:16<12:44,  1.12it/s]\u001b[A\n",
            "Iteration:  15%|█▌        | 156/1015 [02:17<12:44,  1.12it/s]\u001b[A\n",
            "Iteration:  15%|█▌        | 157/1015 [02:18<12:42,  1.13it/s]\u001b[A\n",
            "Iteration:  16%|█▌        | 158/1015 [02:19<12:42,  1.12it/s]\u001b[A\n",
            "Iteration:  16%|█▌        | 159/1015 [02:20<12:42,  1.12it/s]\u001b[A\n",
            "Iteration:  16%|█▌        | 160/1015 [02:21<12:40,  1.12it/s]\u001b[A\n",
            "Iteration:  16%|█▌        | 161/1015 [02:22<12:40,  1.12it/s]\u001b[A\n",
            "Iteration:  16%|█▌        | 162/1015 [02:22<12:39,  1.12it/s]\u001b[A\n",
            "Iteration:  16%|█▌        | 163/1015 [02:23<12:38,  1.12it/s]\u001b[A\n",
            "Iteration:  16%|█▌        | 164/1015 [02:24<12:38,  1.12it/s]\u001b[A\n",
            "Iteration:  16%|█▋        | 165/1015 [02:25<12:37,  1.12it/s]\u001b[A\n",
            "Iteration:  16%|█▋        | 166/1015 [02:26<12:37,  1.12it/s]\u001b[A\n",
            "Iteration:  16%|█▋        | 167/1015 [02:27<12:35,  1.12it/s]\u001b[A\n",
            "Iteration:  17%|█▋        | 168/1015 [02:28<12:35,  1.12it/s]\u001b[A\n",
            "Iteration:  17%|█▋        | 169/1015 [02:29<12:35,  1.12it/s]\u001b[A\n",
            "Iteration:  17%|█▋        | 170/1015 [02:30<12:35,  1.12it/s]\u001b[A\n",
            "Iteration:  17%|█▋        | 171/1015 [02:30<12:33,  1.12it/s]\u001b[A\n",
            "Iteration:  17%|█▋        | 172/1015 [02:31<12:33,  1.12it/s]\u001b[A\n",
            "Iteration:  17%|█▋        | 173/1015 [02:32<12:33,  1.12it/s]\u001b[A\n",
            "Iteration:  17%|█▋        | 174/1015 [02:33<12:34,  1.11it/s]\u001b[A\n",
            "Iteration:  17%|█▋        | 175/1015 [02:34<12:32,  1.12it/s]\u001b[A\n",
            "Iteration:  17%|█▋        | 176/1015 [02:35<12:30,  1.12it/s]\u001b[A\n",
            "Iteration:  17%|█▋        | 177/1015 [02:36<12:28,  1.12it/s]\u001b[A\n",
            "Iteration:  18%|█▊        | 178/1015 [02:37<12:28,  1.12it/s]\u001b[A\n",
            "Iteration:  18%|█▊        | 179/1015 [02:38<12:28,  1.12it/s]\u001b[A\n",
            "Iteration:  18%|█▊        | 180/1015 [02:38<12:26,  1.12it/s]\u001b[A\n",
            "Iteration:  18%|█▊        | 181/1015 [02:39<12:26,  1.12it/s]\u001b[A\n",
            "Iteration:  18%|█▊        | 182/1015 [02:40<12:25,  1.12it/s]\u001b[A\n",
            "Iteration:  18%|█▊        | 183/1015 [02:41<12:21,  1.12it/s]\u001b[A\n",
            "Iteration:  18%|█▊        | 184/1015 [02:42<12:20,  1.12it/s]\u001b[A\n",
            "Iteration:  18%|█▊        | 185/1015 [02:43<12:19,  1.12it/s]\u001b[A\n",
            "Iteration:  18%|█▊        | 186/1015 [02:44<12:18,  1.12it/s]\u001b[A\n",
            "Iteration:  18%|█▊        | 187/1015 [02:45<12:18,  1.12it/s]\u001b[A\n",
            "Iteration:  19%|█▊        | 188/1015 [02:46<12:17,  1.12it/s]\u001b[A\n",
            "Iteration:  19%|█▊        | 189/1015 [02:47<12:16,  1.12it/s]\u001b[A\n",
            "Iteration:  19%|█▊        | 190/1015 [02:47<12:15,  1.12it/s]\u001b[A\n",
            "Iteration:  19%|█▉        | 191/1015 [02:48<12:14,  1.12it/s]\u001b[A\n",
            "Iteration:  19%|█▉        | 192/1015 [02:49<12:13,  1.12it/s]\u001b[A\n",
            "Iteration:  19%|█▉        | 193/1015 [02:50<12:13,  1.12it/s]\u001b[A\n",
            "Iteration:  19%|█▉        | 194/1015 [02:51<12:10,  1.12it/s]\u001b[A\n",
            "Iteration:  19%|█▉        | 195/1015 [02:52<12:09,  1.12it/s]\u001b[A\n",
            "Iteration:  19%|█▉        | 196/1015 [02:53<12:07,  1.13it/s]\u001b[A\n",
            "Iteration:  19%|█▉        | 197/1015 [02:54<12:06,  1.13it/s]\u001b[A\n",
            "Iteration:  20%|█▉        | 198/1015 [02:55<12:05,  1.13it/s]\u001b[A\n",
            "Iteration:  20%|█▉        | 199/1015 [02:55<12:03,  1.13it/s]\u001b[A\n",
            "Iteration:  20%|█▉        | 200/1015 [02:56<12:02,  1.13it/s]\u001b[A\n",
            "Iteration:  20%|█▉        | 201/1015 [02:57<12:02,  1.13it/s]\u001b[A\n",
            "Iteration:  20%|█▉        | 202/1015 [02:58<12:01,  1.13it/s]\u001b[A\n",
            "Iteration:  20%|██        | 203/1015 [02:59<12:00,  1.13it/s]\u001b[A\n",
            "Iteration:  20%|██        | 204/1015 [03:00<12:00,  1.13it/s]\u001b[A\n",
            "Iteration:  20%|██        | 205/1015 [03:01<11:59,  1.13it/s]\u001b[A\n",
            "Iteration:  20%|██        | 206/1015 [03:02<11:58,  1.13it/s]\u001b[A\n",
            "Iteration:  20%|██        | 207/1015 [03:03<11:57,  1.13it/s]\u001b[A\n",
            "Iteration:  20%|██        | 208/1015 [03:03<11:56,  1.13it/s]\u001b[A\n",
            "Iteration:  21%|██        | 209/1015 [03:04<11:55,  1.13it/s]\u001b[A\n",
            "Iteration:  21%|██        | 210/1015 [03:05<11:54,  1.13it/s]\u001b[A\n",
            "Iteration:  21%|██        | 211/1015 [03:06<11:54,  1.12it/s]\u001b[A\n",
            "Iteration:  21%|██        | 212/1015 [03:07<11:51,  1.13it/s]\u001b[A\n",
            "Iteration:  21%|██        | 213/1015 [03:08<11:50,  1.13it/s]\u001b[A\n",
            "Iteration:  21%|██        | 214/1015 [03:09<11:50,  1.13it/s]\u001b[A\n",
            "Iteration:  21%|██        | 215/1015 [03:10<11:49,  1.13it/s]\u001b[A\n",
            "Iteration:  21%|██▏       | 216/1015 [03:10<11:49,  1.13it/s]\u001b[A\n",
            "Iteration:  21%|██▏       | 217/1015 [03:11<11:48,  1.13it/s]\u001b[A\n",
            "Iteration:  21%|██▏       | 218/1015 [03:12<11:48,  1.13it/s]\u001b[A\n",
            "Iteration:  22%|██▏       | 219/1015 [03:13<11:46,  1.13it/s]\u001b[A\n",
            "Iteration:  22%|██▏       | 220/1015 [03:14<11:43,  1.13it/s]\u001b[A\n",
            "Iteration:  22%|██▏       | 221/1015 [03:15<11:42,  1.13it/s]\u001b[A\n",
            "Iteration:  22%|██▏       | 222/1015 [03:16<11:44,  1.13it/s]\u001b[A\n",
            "Iteration:  22%|██▏       | 223/1015 [03:17<11:42,  1.13it/s]\u001b[A\n",
            "Iteration:  22%|██▏       | 224/1015 [03:18<11:43,  1.12it/s]\u001b[A\n",
            "Iteration:  22%|██▏       | 225/1015 [03:18<11:41,  1.13it/s]\u001b[A\n",
            "Iteration:  22%|██▏       | 226/1015 [03:19<11:40,  1.13it/s]\u001b[A\n",
            "Iteration:  22%|██▏       | 227/1015 [03:20<11:38,  1.13it/s]\u001b[A\n",
            "Iteration:  22%|██▏       | 228/1015 [03:21<11:37,  1.13it/s]\u001b[A\n",
            "Iteration:  23%|██▎       | 229/1015 [03:22<11:37,  1.13it/s]\u001b[A\n",
            "Iteration:  23%|██▎       | 230/1015 [03:23<11:37,  1.13it/s]\u001b[A\n",
            "Iteration:  23%|██▎       | 231/1015 [03:24<11:32,  1.13it/s]\u001b[A\n",
            "Iteration:  23%|██▎       | 232/1015 [03:25<11:33,  1.13it/s]\u001b[A\n",
            "Iteration:  23%|██▎       | 233/1015 [03:26<11:32,  1.13it/s]\u001b[A\n",
            "Iteration:  23%|██▎       | 234/1015 [03:26<11:33,  1.13it/s]\u001b[A\n",
            "Iteration:  23%|██▎       | 235/1015 [03:27<11:32,  1.13it/s]\u001b[A\n",
            "Iteration:  23%|██▎       | 236/1015 [03:28<11:29,  1.13it/s]\u001b[A\n",
            "Iteration:  23%|██▎       | 237/1015 [03:29<11:29,  1.13it/s]\u001b[A\n",
            "Iteration:  23%|██▎       | 238/1015 [03:30<11:31,  1.12it/s]\u001b[A\n",
            "Iteration:  24%|██▎       | 239/1015 [03:31<11:28,  1.13it/s]\u001b[A\n",
            "Iteration:  24%|██▎       | 240/1015 [03:32<11:25,  1.13it/s]\u001b[A\n",
            "Iteration:  24%|██▎       | 241/1015 [03:33<11:25,  1.13it/s]\u001b[A\n",
            "Iteration:  24%|██▍       | 242/1015 [03:34<11:25,  1.13it/s]\u001b[A\n",
            "Iteration:  24%|██▍       | 243/1015 [03:34<11:23,  1.13it/s]\u001b[A\n",
            "Iteration:  24%|██▍       | 244/1015 [03:35<11:21,  1.13it/s]\u001b[A\n",
            "Iteration:  24%|██▍       | 245/1015 [03:36<11:20,  1.13it/s]\u001b[A\n",
            "Iteration:  24%|██▍       | 246/1015 [03:37<11:20,  1.13it/s]\u001b[A\n",
            "Iteration:  24%|██▍       | 247/1015 [03:38<11:17,  1.13it/s]\u001b[A\n",
            "Iteration:  24%|██▍       | 248/1015 [03:39<11:16,  1.13it/s]\u001b[A\n",
            "Iteration:  25%|██▍       | 249/1015 [03:40<11:14,  1.14it/s]\u001b[A\n",
            "Iteration:  25%|██▍       | 250/1015 [03:41<11:14,  1.13it/s]\u001b[A\n",
            "Iteration:  25%|██▍       | 251/1015 [03:41<11:14,  1.13it/s]\u001b[A\n",
            "Iteration:  25%|██▍       | 252/1015 [03:42<11:12,  1.13it/s]\u001b[A\n",
            "Iteration:  25%|██▍       | 253/1015 [03:43<11:14,  1.13it/s]\u001b[A\n",
            "Iteration:  25%|██▌       | 254/1015 [03:44<11:10,  1.13it/s]\u001b[A\n",
            "Iteration:  25%|██▌       | 255/1015 [03:45<11:10,  1.13it/s]\u001b[A\n",
            "Iteration:  25%|██▌       | 256/1015 [03:46<11:10,  1.13it/s]\u001b[A\n",
            "Iteration:  25%|██▌       | 257/1015 [03:47<11:10,  1.13it/s]\u001b[A\n",
            "Iteration:  25%|██▌       | 258/1015 [03:48<11:07,  1.13it/s]\u001b[A\n",
            "Iteration:  26%|██▌       | 259/1015 [03:49<11:07,  1.13it/s]\u001b[A\n",
            "Iteration:  26%|██▌       | 260/1015 [03:49<11:05,  1.13it/s]\u001b[A\n",
            "Iteration:  26%|██▌       | 261/1015 [03:50<11:05,  1.13it/s]\u001b[A\n",
            "Iteration:  26%|██▌       | 262/1015 [03:51<11:03,  1.13it/s]\u001b[A\n",
            "Iteration:  26%|██▌       | 263/1015 [03:52<11:02,  1.13it/s]\u001b[A\n",
            "Iteration:  26%|██▌       | 264/1015 [03:53<11:00,  1.14it/s]\u001b[A\n",
            "Iteration:  26%|██▌       | 265/1015 [03:54<11:01,  1.13it/s]\u001b[A\n",
            "Iteration:  26%|██▌       | 266/1015 [03:55<11:01,  1.13it/s]\u001b[A\n",
            "Iteration:  26%|██▋       | 267/1015 [03:56<11:00,  1.13it/s]\u001b[A\n",
            "Iteration:  26%|██▋       | 268/1015 [03:56<10:59,  1.13it/s]\u001b[A\n",
            "Iteration:  27%|██▋       | 269/1015 [03:57<10:57,  1.13it/s]\u001b[A\n",
            "Iteration:  27%|██▋       | 270/1015 [03:58<10:57,  1.13it/s]\u001b[A\n",
            "Iteration:  27%|██▋       | 271/1015 [03:59<10:57,  1.13it/s]\u001b[A\n",
            "Iteration:  27%|██▋       | 272/1015 [04:00<10:56,  1.13it/s]\u001b[A\n",
            "Iteration:  27%|██▋       | 273/1015 [04:01<10:53,  1.14it/s]\u001b[A\n",
            "Iteration:  27%|██▋       | 274/1015 [04:02<10:53,  1.13it/s]\u001b[A\n",
            "Iteration:  27%|██▋       | 275/1015 [04:03<10:52,  1.13it/s]\u001b[A\n",
            "Iteration:  27%|██▋       | 276/1015 [04:04<10:52,  1.13it/s]\u001b[A\n",
            "Iteration:  27%|██▋       | 277/1015 [04:04<10:52,  1.13it/s]\u001b[A\n",
            "Iteration:  27%|██▋       | 278/1015 [04:05<10:53,  1.13it/s]\u001b[A\n",
            "Iteration:  27%|██▋       | 279/1015 [04:06<10:53,  1.13it/s]\u001b[A\n",
            "Iteration:  28%|██▊       | 280/1015 [04:07<10:52,  1.13it/s]\u001b[A\n",
            "Iteration:  28%|██▊       | 281/1015 [04:08<10:52,  1.13it/s]\u001b[A\n",
            "Iteration:  28%|██▊       | 282/1015 [04:09<10:51,  1.12it/s]\u001b[A\n",
            "Iteration:  28%|██▊       | 283/1015 [04:10<10:49,  1.13it/s]\u001b[A\n",
            "Iteration:  28%|██▊       | 284/1015 [04:11<10:47,  1.13it/s]\u001b[A\n",
            "Iteration:  28%|██▊       | 285/1015 [04:12<10:48,  1.13it/s]\u001b[A\n",
            "Iteration:  28%|██▊       | 286/1015 [04:12<10:47,  1.13it/s]\u001b[A\n",
            "Iteration:  28%|██▊       | 287/1015 [04:13<10:47,  1.12it/s]\u001b[A\n",
            "Iteration:  28%|██▊       | 288/1015 [04:14<10:47,  1.12it/s]\u001b[A\n",
            "Iteration:  28%|██▊       | 289/1015 [04:15<10:46,  1.12it/s]\u001b[A\n",
            "Iteration:  29%|██▊       | 290/1015 [04:16<10:45,  1.12it/s]\u001b[A\n",
            "Iteration:  29%|██▊       | 291/1015 [04:17<10:44,  1.12it/s]\u001b[A\n",
            "Iteration:  29%|██▉       | 292/1015 [04:18<10:44,  1.12it/s]\u001b[A\n",
            "Iteration:  29%|██▉       | 293/1015 [04:19<10:42,  1.12it/s]\u001b[A\n",
            "Iteration:  29%|██▉       | 294/1015 [04:20<10:40,  1.13it/s]\u001b[A\n",
            "Iteration:  29%|██▉       | 295/1015 [04:20<10:40,  1.12it/s]\u001b[A\n",
            "Iteration:  29%|██▉       | 296/1015 [04:21<10:40,  1.12it/s]\u001b[A\n",
            "Iteration:  29%|██▉       | 297/1015 [04:22<10:39,  1.12it/s]\u001b[A\n",
            "Iteration:  29%|██▉       | 298/1015 [04:23<10:39,  1.12it/s]\u001b[A\n",
            "Iteration:  29%|██▉       | 299/1015 [04:24<10:38,  1.12it/s]\u001b[A\n",
            "Iteration:  30%|██▉       | 300/1015 [04:25<10:38,  1.12it/s]\u001b[A\n",
            "Iteration:  30%|██▉       | 301/1015 [04:26<10:37,  1.12it/s]\u001b[A\n",
            "Iteration:  30%|██▉       | 302/1015 [04:27<10:36,  1.12it/s]\u001b[A\n",
            "Iteration:  30%|██▉       | 303/1015 [04:28<10:36,  1.12it/s]\u001b[A\n",
            "Iteration:  30%|██▉       | 304/1015 [04:28<10:36,  1.12it/s]\u001b[A\n",
            "Iteration:  30%|███       | 305/1015 [04:29<10:34,  1.12it/s]\u001b[A\n",
            "Iteration:  30%|███       | 306/1015 [04:30<10:35,  1.12it/s]\u001b[A\n",
            "Iteration:  30%|███       | 307/1015 [04:31<10:33,  1.12it/s]\u001b[A\n",
            "Iteration:  30%|███       | 308/1015 [04:32<10:32,  1.12it/s]\u001b[A\n",
            "Iteration:  30%|███       | 309/1015 [04:33<10:32,  1.12it/s]\u001b[A\n",
            "Iteration:  31%|███       | 310/1015 [04:34<10:32,  1.12it/s]\u001b[A\n",
            "Iteration:  31%|███       | 311/1015 [04:35<10:29,  1.12it/s]\u001b[A\n",
            "Iteration:  31%|███       | 312/1015 [04:36<10:28,  1.12it/s]\u001b[A\n",
            "Iteration:  31%|███       | 313/1015 [04:37<10:26,  1.12it/s]\u001b[A\n",
            "Iteration:  31%|███       | 314/1015 [04:37<10:26,  1.12it/s]\u001b[A\n",
            "Iteration:  31%|███       | 315/1015 [04:38<10:25,  1.12it/s]\u001b[A\n",
            "Iteration:  31%|███       | 316/1015 [04:39<10:25,  1.12it/s]\u001b[A\n",
            "Iteration:  31%|███       | 317/1015 [04:40<10:24,  1.12it/s]\u001b[A\n",
            "Iteration:  31%|███▏      | 318/1015 [04:41<10:21,  1.12it/s]\u001b[A\n",
            "Iteration:  31%|███▏      | 319/1015 [04:42<10:20,  1.12it/s]\u001b[A\n",
            "Iteration:  32%|███▏      | 320/1015 [04:43<10:20,  1.12it/s]\u001b[A\n",
            "Iteration:  32%|███▏      | 321/1015 [04:44<10:20,  1.12it/s]\u001b[A\n",
            "Iteration:  32%|███▏      | 322/1015 [04:45<10:19,  1.12it/s]\u001b[A\n",
            "Iteration:  32%|███▏      | 323/1015 [04:45<10:19,  1.12it/s]\u001b[A\n",
            "Iteration:  32%|███▏      | 324/1015 [04:46<10:17,  1.12it/s]\u001b[A\n",
            "Iteration:  32%|███▏      | 325/1015 [04:47<10:16,  1.12it/s]\u001b[A\n",
            "Iteration:  32%|███▏      | 326/1015 [04:48<10:15,  1.12it/s]\u001b[A\n",
            "Iteration:  32%|███▏      | 327/1015 [04:49<10:15,  1.12it/s]\u001b[A\n",
            "Iteration:  32%|███▏      | 328/1015 [04:50<10:13,  1.12it/s]\u001b[A\n",
            "Iteration:  32%|███▏      | 329/1015 [04:51<10:13,  1.12it/s]\u001b[A\n",
            "Iteration:  33%|███▎      | 330/1015 [04:52<10:12,  1.12it/s]\u001b[A\n",
            "Iteration:  33%|███▎      | 331/1015 [04:53<10:10,  1.12it/s]\u001b[A\n",
            "Iteration:  33%|███▎      | 332/1015 [04:54<10:10,  1.12it/s]\u001b[A\n",
            "Iteration:  33%|███▎      | 333/1015 [04:54<10:09,  1.12it/s]\u001b[A\n",
            "Iteration:  33%|███▎      | 334/1015 [04:55<10:08,  1.12it/s]\u001b[A\n",
            "Iteration:  33%|███▎      | 335/1015 [04:56<10:08,  1.12it/s]\u001b[A\n",
            "Iteration:  33%|███▎      | 336/1015 [04:57<10:06,  1.12it/s]\u001b[A\n",
            "Iteration:  33%|███▎      | 337/1015 [04:58<10:05,  1.12it/s]\u001b[A\n",
            "Iteration:  33%|███▎      | 338/1015 [04:59<10:04,  1.12it/s]\u001b[A\n",
            "Iteration:  33%|███▎      | 339/1015 [05:00<10:01,  1.12it/s]\u001b[A\n",
            "Iteration:  33%|███▎      | 340/1015 [05:01<09:59,  1.13it/s]\u001b[A\n",
            "Iteration:  34%|███▎      | 341/1015 [05:02<10:00,  1.12it/s]\u001b[A\n",
            "Iteration:  34%|███▎      | 342/1015 [05:02<09:59,  1.12it/s]\u001b[A\n",
            "Iteration:  34%|███▍      | 343/1015 [05:03<09:58,  1.12it/s]\u001b[A\n",
            "Iteration:  34%|███▍      | 344/1015 [05:04<09:56,  1.13it/s]\u001b[A\n",
            "Iteration:  34%|███▍      | 345/1015 [05:05<09:56,  1.12it/s]\u001b[A\n",
            "Iteration:  34%|███▍      | 346/1015 [05:06<09:55,  1.12it/s]\u001b[A\n",
            "Iteration:  34%|███▍      | 347/1015 [05:07<09:54,  1.12it/s]\u001b[A\n",
            "Iteration:  34%|███▍      | 348/1015 [05:08<09:52,  1.13it/s]\u001b[A\n",
            "Iteration:  34%|███▍      | 349/1015 [05:09<09:50,  1.13it/s]\u001b[A\n",
            "Iteration:  34%|███▍      | 350/1015 [05:10<09:49,  1.13it/s]\u001b[A\n",
            "Iteration:  35%|███▍      | 351/1015 [05:10<09:49,  1.13it/s]\u001b[A\n",
            "Iteration:  35%|███▍      | 352/1015 [05:11<09:49,  1.12it/s]\u001b[A\n",
            "Iteration:  35%|███▍      | 353/1015 [05:12<09:48,  1.12it/s]\u001b[A\n",
            "Iteration:  35%|███▍      | 354/1015 [05:13<09:46,  1.13it/s]\u001b[A\n",
            "Iteration:  35%|███▍      | 355/1015 [05:14<09:45,  1.13it/s]\u001b[A\n",
            "Iteration:  35%|███▌      | 356/1015 [05:15<09:44,  1.13it/s]\u001b[A\n",
            "Iteration:  35%|███▌      | 357/1015 [05:16<09:41,  1.13it/s]\u001b[A\n",
            "Iteration:  35%|███▌      | 358/1015 [05:17<09:41,  1.13it/s]\u001b[A\n",
            "Iteration:  35%|███▌      | 359/1015 [05:18<09:42,  1.13it/s]\u001b[A\n",
            "Iteration:  35%|███▌      | 360/1015 [05:18<09:40,  1.13it/s]\u001b[A\n",
            "Iteration:  36%|███▌      | 361/1015 [05:19<09:39,  1.13it/s]\u001b[A\n",
            "Iteration:  36%|███▌      | 362/1015 [05:20<09:38,  1.13it/s]\u001b[A\n",
            "Iteration:  36%|███▌      | 363/1015 [05:21<09:36,  1.13it/s]\u001b[A\n",
            "Iteration:  36%|███▌      | 364/1015 [05:22<09:36,  1.13it/s]\u001b[A\n",
            "Iteration:  36%|███▌      | 365/1015 [05:23<09:35,  1.13it/s]\u001b[A\n",
            "Iteration:  36%|███▌      | 366/1015 [05:24<09:34,  1.13it/s]\u001b[A\n",
            "Iteration:  36%|███▌      | 367/1015 [05:25<09:32,  1.13it/s]\u001b[A\n",
            "Iteration:  36%|███▋      | 368/1015 [05:25<09:32,  1.13it/s]\u001b[A\n",
            "Iteration:  36%|███▋      | 369/1015 [05:26<09:32,  1.13it/s]\u001b[A\n",
            "Iteration:  36%|███▋      | 370/1015 [05:27<09:32,  1.13it/s]\u001b[A\n",
            "Iteration:  37%|███▋      | 371/1015 [05:28<09:31,  1.13it/s]\u001b[A\n",
            "Iteration:  37%|███▋      | 372/1015 [05:29<09:31,  1.13it/s]\u001b[A\n",
            "Iteration:  37%|███▋      | 373/1015 [05:30<09:29,  1.13it/s]\u001b[A\n",
            "Iteration:  37%|███▋      | 374/1015 [05:31<09:29,  1.13it/s]\u001b[A\n",
            "Iteration:  37%|███▋      | 375/1015 [05:32<09:26,  1.13it/s]\u001b[A\n",
            "Iteration:  37%|███▋      | 376/1015 [05:33<09:26,  1.13it/s]\u001b[A\n",
            "Iteration:  37%|███▋      | 377/1015 [05:33<09:25,  1.13it/s]\u001b[A\n",
            "Iteration:  37%|███▋      | 378/1015 [05:34<09:24,  1.13it/s]\u001b[A\n",
            "Iteration:  37%|███▋      | 379/1015 [05:35<09:23,  1.13it/s]\u001b[A\n",
            "Iteration:  37%|███▋      | 380/1015 [05:36<09:22,  1.13it/s]\u001b[A\n",
            "Iteration:  38%|███▊      | 381/1015 [05:37<09:21,  1.13it/s]\u001b[A\n",
            "Iteration:  38%|███▊      | 382/1015 [05:38<09:19,  1.13it/s]\u001b[A\n",
            "Iteration:  38%|███▊      | 383/1015 [05:39<09:18,  1.13it/s]\u001b[A\n",
            "Iteration:  38%|███▊      | 384/1015 [05:40<09:17,  1.13it/s]\u001b[A\n",
            "Iteration:  38%|███▊      | 385/1015 [05:41<09:17,  1.13it/s]\u001b[A\n",
            "Iteration:  38%|███▊      | 386/1015 [05:41<09:16,  1.13it/s]\u001b[A\n",
            "Iteration:  38%|███▊      | 387/1015 [05:42<09:15,  1.13it/s]\u001b[A\n",
            "Iteration:  38%|███▊      | 388/1015 [05:43<09:14,  1.13it/s]\u001b[A\n",
            "Iteration:  38%|███▊      | 389/1015 [05:44<09:14,  1.13it/s]\u001b[A\n",
            "Iteration:  38%|███▊      | 390/1015 [05:45<09:14,  1.13it/s]\u001b[A\n",
            "Iteration:  39%|███▊      | 391/1015 [05:46<09:14,  1.13it/s]\u001b[A\n",
            "Iteration:  39%|███▊      | 392/1015 [05:47<09:12,  1.13it/s]\u001b[A\n",
            "Iteration:  39%|███▊      | 393/1015 [05:48<09:10,  1.13it/s]\u001b[A\n",
            "Iteration:  39%|███▉      | 394/1015 [05:49<09:09,  1.13it/s]\u001b[A\n",
            "Iteration:  39%|███▉      | 395/1015 [05:49<09:08,  1.13it/s]\u001b[A\n",
            "Iteration:  39%|███▉      | 396/1015 [05:50<09:08,  1.13it/s]\u001b[A\n",
            "Iteration:  39%|███▉      | 397/1015 [05:51<09:06,  1.13it/s]\u001b[A\n",
            "Iteration:  39%|███▉      | 398/1015 [05:52<09:06,  1.13it/s]\u001b[A\n",
            "Iteration:  39%|███▉      | 399/1015 [05:53<09:04,  1.13it/s]\u001b[A\n",
            "Iteration:  39%|███▉      | 400/1015 [05:54<09:03,  1.13it/s]\u001b[A\n",
            "Iteration:  40%|███▉      | 401/1015 [05:55<09:03,  1.13it/s]\u001b[A\n",
            "Iteration:  40%|███▉      | 402/1015 [05:56<09:02,  1.13it/s]\u001b[A\n",
            "Iteration:  40%|███▉      | 403/1015 [05:56<09:01,  1.13it/s]\u001b[A\n",
            "Iteration:  40%|███▉      | 404/1015 [05:57<09:02,  1.13it/s]\u001b[A\n",
            "Iteration:  40%|███▉      | 405/1015 [05:58<09:02,  1.12it/s]\u001b[A\n",
            "Iteration:  40%|████      | 406/1015 [05:59<09:01,  1.12it/s]\u001b[A\n",
            "Iteration:  40%|████      | 407/1015 [06:00<09:00,  1.13it/s]\u001b[A\n",
            "Iteration:  40%|████      | 408/1015 [06:01<08:57,  1.13it/s]\u001b[A\n",
            "Iteration:  40%|████      | 409/1015 [06:02<08:56,  1.13it/s]\u001b[A\n",
            "Iteration:  40%|████      | 410/1015 [06:03<08:56,  1.13it/s]\u001b[A\n",
            "Iteration:  40%|████      | 411/1015 [06:04<08:55,  1.13it/s]\u001b[A\n",
            "Iteration:  41%|████      | 412/1015 [06:04<08:56,  1.12it/s]\u001b[A\n",
            "Iteration:  41%|████      | 413/1015 [06:05<08:55,  1.13it/s]\u001b[A\n",
            "Iteration:  41%|████      | 414/1015 [06:06<08:54,  1.12it/s]\u001b[A\n",
            "Iteration:  41%|████      | 415/1015 [06:07<08:53,  1.13it/s]\u001b[A\n",
            "Iteration:  41%|████      | 416/1015 [06:08<08:52,  1.13it/s]\u001b[A\n",
            "Iteration:  41%|████      | 417/1015 [06:09<08:51,  1.12it/s]\u001b[A\n",
            "Iteration:  41%|████      | 418/1015 [06:10<08:49,  1.13it/s]\u001b[A\n",
            "Iteration:  41%|████▏     | 419/1015 [06:11<08:48,  1.13it/s]\u001b[A\n",
            "Iteration:  41%|████▏     | 420/1015 [06:12<08:47,  1.13it/s]\u001b[A\n",
            "Iteration:  41%|████▏     | 421/1015 [06:12<08:47,  1.13it/s]\u001b[A\n",
            "Iteration:  42%|████▏     | 422/1015 [06:13<08:45,  1.13it/s]\u001b[A\n",
            "Iteration:  42%|████▏     | 423/1015 [06:14<08:46,  1.12it/s]\u001b[A\n",
            "Iteration:  42%|████▏     | 424/1015 [06:15<08:45,  1.13it/s]\u001b[A\n",
            "Iteration:  42%|████▏     | 425/1015 [06:16<08:43,  1.13it/s]\u001b[A\n",
            "Iteration:  42%|████▏     | 426/1015 [06:17<08:41,  1.13it/s]\u001b[A\n",
            "Iteration:  42%|████▏     | 427/1015 [06:18<08:41,  1.13it/s]\u001b[A\n",
            "Iteration:  42%|████▏     | 428/1015 [06:19<08:38,  1.13it/s]\u001b[A\n",
            "Iteration:  42%|████▏     | 429/1015 [06:20<08:39,  1.13it/s]\u001b[A\n",
            "Iteration:  42%|████▏     | 430/1015 [06:20<08:39,  1.13it/s]\u001b[A\n",
            "Iteration:  42%|████▏     | 431/1015 [06:21<08:37,  1.13it/s]\u001b[A\n",
            "Iteration:  43%|████▎     | 432/1015 [06:22<08:36,  1.13it/s]\u001b[A\n",
            "Iteration:  43%|████▎     | 433/1015 [06:23<08:35,  1.13it/s]\u001b[A\n",
            "Iteration:  43%|████▎     | 434/1015 [06:24<08:34,  1.13it/s]\u001b[A\n",
            "Iteration:  43%|████▎     | 435/1015 [06:25<08:34,  1.13it/s]\u001b[A\n",
            "Iteration:  43%|████▎     | 436/1015 [06:26<08:34,  1.13it/s]\u001b[A\n",
            "Iteration:  43%|████▎     | 437/1015 [06:27<08:32,  1.13it/s]\u001b[A\n",
            "Iteration:  43%|████▎     | 438/1015 [06:28<08:31,  1.13it/s]\u001b[A\n",
            "Iteration:  43%|████▎     | 439/1015 [06:28<08:29,  1.13it/s]\u001b[A\n",
            "Iteration:  43%|████▎     | 440/1015 [06:29<08:30,  1.13it/s]\u001b[A\n",
            "Iteration:  43%|████▎     | 441/1015 [06:30<08:29,  1.13it/s]\u001b[A\n",
            "Iteration:  44%|████▎     | 442/1015 [06:31<08:27,  1.13it/s]\u001b[A\n",
            "Iteration:  44%|████▎     | 443/1015 [06:32<08:27,  1.13it/s]\u001b[A\n",
            "Iteration:  44%|████▎     | 444/1015 [06:33<08:25,  1.13it/s]\u001b[A\n",
            "Iteration:  44%|████▍     | 445/1015 [06:34<08:24,  1.13it/s]\u001b[A\n",
            "Iteration:  44%|████▍     | 446/1015 [06:35<08:24,  1.13it/s]\u001b[A\n",
            "Iteration:  44%|████▍     | 447/1015 [06:36<08:24,  1.13it/s]\u001b[A\n",
            "Iteration:  44%|████▍     | 448/1015 [06:36<08:23,  1.13it/s]\u001b[A\n",
            "Iteration:  44%|████▍     | 449/1015 [06:37<08:22,  1.13it/s]\u001b[A\n",
            "Iteration:  44%|████▍     | 450/1015 [06:38<08:22,  1.13it/s]\u001b[A\n",
            "Iteration:  44%|████▍     | 451/1015 [06:39<08:22,  1.12it/s]\u001b[A\n",
            "Iteration:  45%|████▍     | 452/1015 [06:40<08:21,  1.12it/s]\u001b[A\n",
            "Iteration:  45%|████▍     | 453/1015 [06:41<08:20,  1.12it/s]\u001b[A\n",
            "Iteration:  45%|████▍     | 454/1015 [06:42<08:18,  1.12it/s]\u001b[A\n",
            "Iteration:  45%|████▍     | 455/1015 [06:43<08:18,  1.12it/s]\u001b[A\n",
            "Iteration:  45%|████▍     | 456/1015 [06:44<08:17,  1.12it/s]\u001b[A\n",
            "Iteration:  45%|████▌     | 457/1015 [06:44<08:14,  1.13it/s]\u001b[A\n",
            "Iteration:  45%|████▌     | 458/1015 [06:45<08:14,  1.13it/s]\u001b[A\n",
            "Iteration:  45%|████▌     | 459/1015 [06:46<08:14,  1.13it/s]\u001b[A\n",
            "Iteration:  45%|████▌     | 460/1015 [06:47<08:13,  1.12it/s]\u001b[A\n",
            "Iteration:  45%|████▌     | 461/1015 [06:48<08:13,  1.12it/s]\u001b[A\n",
            "Iteration:  46%|████▌     | 462/1015 [06:49<08:13,  1.12it/s]\u001b[A\n",
            "Iteration:  46%|████▌     | 463/1015 [06:50<08:10,  1.12it/s]\u001b[A\n",
            "Iteration:  46%|████▌     | 464/1015 [06:51<08:09,  1.13it/s]\u001b[A\n",
            "Iteration:  46%|████▌     | 465/1015 [06:52<08:09,  1.12it/s]\u001b[A\n",
            "Iteration:  46%|████▌     | 466/1015 [06:52<08:08,  1.12it/s]\u001b[A\n",
            "Iteration:  46%|████▌     | 467/1015 [06:53<08:08,  1.12it/s]\u001b[A\n",
            "Iteration:  46%|████▌     | 468/1015 [06:54<08:05,  1.13it/s]\u001b[A\n",
            "Iteration:  46%|████▌     | 469/1015 [06:55<08:04,  1.13it/s]\u001b[A\n",
            "Iteration:  46%|████▋     | 470/1015 [06:56<08:03,  1.13it/s]\u001b[A\n",
            "Iteration:  46%|████▋     | 471/1015 [06:57<08:02,  1.13it/s]\u001b[A\n",
            "Iteration:  47%|████▋     | 472/1015 [06:58<08:01,  1.13it/s]\u001b[A\n",
            "Iteration:  47%|████▋     | 473/1015 [06:59<08:00,  1.13it/s]\u001b[A\n",
            "Iteration:  47%|████▋     | 474/1015 [07:00<07:59,  1.13it/s]\u001b[A\n",
            "Iteration:  47%|████▋     | 475/1015 [07:00<07:59,  1.13it/s]\u001b[A\n",
            "Iteration:  47%|████▋     | 476/1015 [07:01<07:59,  1.13it/s]\u001b[A\n",
            "Iteration:  47%|████▋     | 477/1015 [07:02<07:58,  1.12it/s]\u001b[A\n",
            "Iteration:  47%|████▋     | 478/1015 [07:03<07:58,  1.12it/s]\u001b[A\n",
            "Iteration:  47%|████▋     | 479/1015 [07:04<07:57,  1.12it/s]\u001b[A\n",
            "Iteration:  47%|████▋     | 480/1015 [07:05<07:57,  1.12it/s]\u001b[A\n",
            "Iteration:  47%|████▋     | 481/1015 [07:06<07:55,  1.12it/s]\u001b[A\n",
            "Iteration:  47%|████▋     | 482/1015 [07:07<07:53,  1.13it/s]\u001b[A\n",
            "Iteration:  48%|████▊     | 483/1015 [07:08<07:52,  1.13it/s]\u001b[A\n",
            "Iteration:  48%|████▊     | 484/1015 [07:08<07:52,  1.12it/s]\u001b[A\n",
            "Iteration:  48%|████▊     | 485/1015 [07:09<07:52,  1.12it/s]\u001b[A\n",
            "Iteration:  48%|████▊     | 486/1015 [07:10<07:52,  1.12it/s]\u001b[A\n",
            "Iteration:  48%|████▊     | 487/1015 [07:11<07:50,  1.12it/s]\u001b[A\n",
            "Iteration:  48%|████▊     | 488/1015 [07:12<07:49,  1.12it/s]\u001b[A\n",
            "Iteration:  48%|████▊     | 489/1015 [07:13<07:48,  1.12it/s]\u001b[A\n",
            "Iteration:  48%|████▊     | 490/1015 [07:14<07:47,  1.12it/s]\u001b[A\n",
            "Iteration:  48%|████▊     | 491/1015 [07:15<07:46,  1.12it/s]\u001b[A\n",
            "Iteration:  48%|████▊     | 492/1015 [07:16<07:45,  1.12it/s]\u001b[A\n",
            "Iteration:  49%|████▊     | 493/1015 [07:16<07:45,  1.12it/s]\u001b[A\n",
            "Iteration:  49%|████▊     | 494/1015 [07:17<07:43,  1.12it/s]\u001b[A\n",
            "Iteration:  49%|████▉     | 495/1015 [07:18<07:42,  1.12it/s]\u001b[A\n",
            "Iteration:  49%|████▉     | 496/1015 [07:19<07:41,  1.12it/s]\u001b[A\n",
            "Iteration:  49%|████▉     | 497/1015 [07:20<07:40,  1.13it/s]\u001b[A\n",
            "Iteration:  49%|████▉     | 498/1015 [07:21<07:39,  1.12it/s]\u001b[A\n",
            "Iteration:  49%|████▉     | 499/1015 [07:22<07:39,  1.12it/s]\u001b[A\n",
            "Iteration:  49%|████▉     | 500/1015 [07:23<07:38,  1.12it/s]\u001b[A\n",
            "Iteration:  49%|████▉     | 501/1015 [07:24<07:36,  1.13it/s]\u001b[A\n",
            "Iteration:  49%|████▉     | 502/1015 [07:24<07:35,  1.13it/s]\u001b[A\n",
            "Iteration:  50%|████▉     | 503/1015 [07:25<07:35,  1.13it/s]\u001b[A\n",
            "Iteration:  50%|████▉     | 504/1015 [07:26<07:35,  1.12it/s]\u001b[A\n",
            "Iteration:  50%|████▉     | 505/1015 [07:27<07:34,  1.12it/s]\u001b[A\n",
            "Iteration:  50%|████▉     | 506/1015 [07:28<07:32,  1.12it/s]\u001b[A\n",
            "Iteration:  50%|████▉     | 507/1015 [07:29<07:32,  1.12it/s]\u001b[A\n",
            "Iteration:  50%|█████     | 508/1015 [07:30<07:31,  1.12it/s]\u001b[A\n",
            "Iteration:  50%|█████     | 509/1015 [07:31<07:30,  1.12it/s]\u001b[A\n",
            "Iteration:  50%|█████     | 510/1015 [07:32<07:29,  1.12it/s]\u001b[A\n",
            "Iteration:  50%|█████     | 511/1015 [07:32<07:28,  1.12it/s]\u001b[A\n",
            "Iteration:  50%|█████     | 512/1015 [07:33<07:27,  1.12it/s]\u001b[A\n",
            "Iteration:  51%|█████     | 513/1015 [07:34<07:26,  1.12it/s]\u001b[A\n",
            "Iteration:  51%|█████     | 514/1015 [07:35<07:26,  1.12it/s]\u001b[A\n",
            "Iteration:  51%|█████     | 515/1015 [07:36<07:25,  1.12it/s]\u001b[A\n",
            "Iteration:  51%|█████     | 516/1015 [07:37<07:24,  1.12it/s]\u001b[A\n",
            "Iteration:  51%|█████     | 517/1015 [07:38<07:24,  1.12it/s]\u001b[A\n",
            "Iteration:  51%|█████     | 518/1015 [07:39<07:22,  1.12it/s]\u001b[A\n",
            "Iteration:  51%|█████     | 519/1015 [07:40<07:22,  1.12it/s]\u001b[A\n",
            "Iteration:  51%|█████     | 520/1015 [07:40<07:20,  1.12it/s]\u001b[A\n",
            "Iteration:  51%|█████▏    | 521/1015 [07:41<07:19,  1.12it/s]\u001b[A\n",
            "Iteration:  51%|█████▏    | 522/1015 [07:42<07:18,  1.12it/s]\u001b[A\n",
            "Iteration:  52%|█████▏    | 523/1015 [07:43<07:18,  1.12it/s]\u001b[A\n",
            "Iteration:  52%|█████▏    | 524/1015 [07:44<07:17,  1.12it/s]\u001b[A\n",
            "Iteration:  52%|█████▏    | 525/1015 [07:45<07:16,  1.12it/s]\u001b[A\n",
            "Iteration:  52%|█████▏    | 526/1015 [07:46<07:15,  1.12it/s]\u001b[A\n",
            "Iteration:  52%|█████▏    | 527/1015 [07:47<07:14,  1.12it/s]\u001b[A\n",
            "Iteration:  52%|█████▏    | 528/1015 [07:48<07:13,  1.12it/s]\u001b[A\n",
            "Iteration:  52%|█████▏    | 529/1015 [07:48<07:12,  1.12it/s]\u001b[A\n",
            "Iteration:  52%|█████▏    | 530/1015 [07:49<07:11,  1.12it/s]\u001b[A\n",
            "Iteration:  52%|█████▏    | 531/1015 [07:50<07:11,  1.12it/s]\u001b[A\n",
            "Iteration:  52%|█████▏    | 532/1015 [07:51<07:10,  1.12it/s]\u001b[A\n",
            "Iteration:  53%|█████▎    | 533/1015 [07:52<07:09,  1.12it/s]\u001b[A\n",
            "Iteration:  53%|█████▎    | 534/1015 [07:53<07:07,  1.12it/s]\u001b[A\n",
            "Iteration:  53%|█████▎    | 535/1015 [07:54<07:08,  1.12it/s]\u001b[A\n",
            "Iteration:  53%|█████▎    | 536/1015 [07:55<07:08,  1.12it/s]\u001b[A\n",
            "Iteration:  53%|█████▎    | 537/1015 [07:56<07:06,  1.12it/s]\u001b[A\n",
            "Iteration:  53%|█████▎    | 538/1015 [07:57<07:05,  1.12it/s]\u001b[A\n",
            "Iteration:  53%|█████▎    | 539/1015 [07:57<07:04,  1.12it/s]\u001b[A\n",
            "Iteration:  53%|█████▎    | 540/1015 [07:58<07:03,  1.12it/s]\u001b[A\n",
            "Iteration:  53%|█████▎    | 541/1015 [07:59<07:02,  1.12it/s]\u001b[A\n",
            "Iteration:  53%|█████▎    | 542/1015 [08:00<07:00,  1.12it/s]\u001b[A\n",
            "Iteration:  53%|█████▎    | 543/1015 [08:01<06:59,  1.12it/s]\u001b[A\n",
            "Iteration:  54%|█████▎    | 544/1015 [08:02<07:00,  1.12it/s]\u001b[A\n",
            "Iteration:  54%|█████▎    | 545/1015 [08:03<06:58,  1.12it/s]\u001b[A\n",
            "Iteration:  54%|█████▍    | 546/1015 [08:04<06:57,  1.12it/s]\u001b[A\n",
            "Iteration:  54%|█████▍    | 547/1015 [08:05<06:56,  1.12it/s]\u001b[A\n",
            "Iteration:  54%|█████▍    | 548/1015 [08:05<06:55,  1.12it/s]\u001b[A\n",
            "Iteration:  54%|█████▍    | 549/1015 [08:06<06:55,  1.12it/s]\u001b[A\n",
            "Iteration:  54%|█████▍    | 550/1015 [08:07<06:54,  1.12it/s]\u001b[A\n",
            "Iteration:  54%|█████▍    | 551/1015 [08:08<06:54,  1.12it/s]\u001b[A\n",
            "Iteration:  54%|█████▍    | 552/1015 [08:09<06:53,  1.12it/s]\u001b[A\n",
            "Iteration:  54%|█████▍    | 553/1015 [08:10<06:51,  1.12it/s]\u001b[A\n",
            "Iteration:  55%|█████▍    | 554/1015 [08:11<06:50,  1.12it/s]\u001b[A\n",
            "Iteration:  55%|█████▍    | 555/1015 [08:12<06:50,  1.12it/s]\u001b[A\n",
            "Iteration:  55%|█████▍    | 556/1015 [08:13<06:49,  1.12it/s]\u001b[A\n",
            "Iteration:  55%|█████▍    | 557/1015 [08:13<06:48,  1.12it/s]\u001b[A\n",
            "Iteration:  55%|█████▍    | 558/1015 [08:14<06:47,  1.12it/s]\u001b[A\n",
            "Iteration:  55%|█████▌    | 559/1015 [08:15<06:46,  1.12it/s]\u001b[A\n",
            "Iteration:  55%|█████▌    | 560/1015 [08:16<06:45,  1.12it/s]\u001b[A\n",
            "Iteration:  55%|█████▌    | 561/1015 [08:17<06:44,  1.12it/s]\u001b[A\n",
            "Iteration:  55%|█████▌    | 562/1015 [08:18<06:44,  1.12it/s]\u001b[A\n",
            "Iteration:  55%|█████▌    | 563/1015 [08:19<06:43,  1.12it/s]\u001b[A\n",
            "Iteration:  56%|█████▌    | 564/1015 [08:20<06:41,  1.12it/s]\u001b[A\n",
            "Iteration:  56%|█████▌    | 565/1015 [08:21<06:41,  1.12it/s]\u001b[A\n",
            "Iteration:  56%|█████▌    | 566/1015 [08:21<06:39,  1.12it/s]\u001b[A\n",
            "Iteration:  56%|█████▌    | 567/1015 [08:22<06:38,  1.12it/s]\u001b[A\n",
            "Iteration:  56%|█████▌    | 568/1015 [08:23<06:38,  1.12it/s]\u001b[A\n",
            "Iteration:  56%|█████▌    | 569/1015 [08:24<06:37,  1.12it/s]\u001b[A\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RWC7rTAJP7hy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "% matplotlib inline\n",
        "\n",
        "import seaborn as sns\n",
        "\n",
        "# Use plot styling from seaborn.\n",
        "sns.set(style='darkgrid')\n",
        "\n",
        "# Increase the plot size and font size.\n",
        "sns.set(font_scale=1.5)\n",
        "plt.rcParams[\"figure.figsize\"] = (12,6)\n",
        "\n",
        "# Plot the learning curve.\n",
        "plt.plot(loss_tab, 'b-o')\n",
        "\n",
        "# Label the plot.\n",
        "plt.title(\"Training loss\")\n",
        "plt.xlabel(\"Epoch\")\n",
        "plt.ylabel(\"Loss\")\n",
        "\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_9vogH6ede6i",
        "colab_type": "text"
      },
      "source": [
        "# **Evaluating the Model**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6xBcVqEw3SH7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def evaluate(model, tokenizer, prefix=\"\"):\n",
        "\n",
        "    if not os.path.exists(output_dir):\n",
        "        os.makedirs(output_dir)\n",
        "\n",
        "    # Note that DistributedSampler samples randomly\n",
        "    eval_sampler = SequentialSampler(dev_dataset)\n",
        "    eval_dataloader = DataLoader(dev_dataset, sampler=eval_sampler, batch_size=eval_batch_size)\n",
        "\n",
        "    # Eval!\n",
        "    print(\"***** Running evaluation *****\")\n",
        "    print(\"  Num examples = \", len(dev_dataset))\n",
        "    print(\"  Batch size = \", eval_batch_size)\n",
        "\n",
        "    all_results = []\n",
        "    start_time = timeit.default_timer()\n",
        "\n",
        "    for batch in tqdm(eval_dataloader, desc=\"Evaluating\"):\n",
        "        model.eval()\n",
        "        batch = tuple(t.to(device) for t in batch)\n",
        "\n",
        "        with torch.no_grad():\n",
        "            inputs = {\n",
        "                \"input_ids\": batch[0],\n",
        "                \"attention_mask\": batch[1],\n",
        "                \"token_type_ids\": batch[2],\n",
        "            }\n",
        "\n",
        "            example_indices = batch[3]\n",
        "            outputs = model(**inputs)\n",
        "\n",
        "        for i, example_index in enumerate(example_indices):\n",
        "            eval_feature = dev_features[example_index.item()]\n",
        "            unique_id = int(eval_feature.unique_id)\n",
        "\n",
        "            output = [to_list(output[i]) for output in outputs]\n",
        "\n",
        "            start_logits, end_logits = output\n",
        "            result = SquadResult(unique_id, start_logits, end_logits)\n",
        "\n",
        "            all_results.append(result)\n",
        "\n",
        "    evalTime = timeit.default_timer() - start_time\n",
        "    print(\"  Evaluation done in total\" , evalTime,\" secs (\", evalTime / len(dev_dataset),\" sec per example)\")\n",
        "\n",
        "    # Compute predictions\n",
        "    output_prediction_file = os.path.join(output_dir, \"predictions_{}.json\".format(prefix))\n",
        "    output_nbest_file = os.path.join(output_dir, \"nbest_predictions_{}.json\".format(prefix))\n",
        "    output_null_log_odds_file = os.path.join(output_dir, \"null_odds_{}.json\".format(prefix))\n",
        "\n",
        "    predictions = compute_predictions_logits(\n",
        "        dev_examples,\n",
        "        dev_features,\n",
        "        all_results,\n",
        "        n_best_size,\n",
        "        max_answer_length,\n",
        "        do_lower_case,\n",
        "        output_prediction_file,\n",
        "        output_nbest_file,\n",
        "        output_null_log_odds_file,\n",
        "        verbose_logging,\n",
        "        version_2_with_negative,\n",
        "        null_score_diff_threshold,\n",
        "        tokenizer,\n",
        "    )\n",
        "\n",
        "    # Compute the F1 and exact scores.\n",
        "    results = squad_evaluate(dev_examples, predictions)\n",
        "    return results"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bad_w6LuEyKD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "results = evaluate(model, tokenizer)\n",
        "print(results)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h_ho6nWBlVg0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%run squaddir/evaluate-v2.0.py squaddir/dev-v2.0.json ./finetuned_squad/predictions_.json"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pexdqfTtCt1D",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Create output directory if needed\n",
        "if not os.path.exists(output_dir):\n",
        "    os.makedirs(output_dir)\n",
        "\n",
        "print(\"Saving model to %s\" % output_dir)\n",
        "\n",
        "# Save a trained model, configuration and tokenizer using `save_pretrained()`.\n",
        "# They can then be reloaded using `from_pretrained()`\n",
        "model_to_save = model.module if hasattr(model, 'module') else model  # Take care of distributed/parallel training\n",
        "model_to_save.save_pretrained(output_dir)\n",
        "tokenizer.save_pretrained(output_dir)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CxBVPcUBdo4W",
        "colab_type": "text"
      },
      "source": [
        "# **Other Stuff and little tests** (not necessarily working)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4w7CoIGOur6W",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# import random\n",
        "\n",
        "# # Note: AdamW is a class from the huggingface library (as opposed to pytorch) \n",
        "# # I believe the 'W' stands for 'Weight Decay fix\"\n",
        "# optimizer = AdamW(model.parameters(),\n",
        "#                   lr = 2e-5, # args.learning_rate - default is 5e-5, our notebook had 2e-5\n",
        "#                   eps = 1e-8 # args.adam_epsilon  - default is 1e-8.\n",
        "#                 )\n",
        "\n",
        "# from transformers import get_linear_schedule_with_warmup\n",
        "\n",
        "# # Number of training epochs (authors recommend between 2 and 4)\n",
        "# epochs = 4\n",
        "\n",
        "# # Total number of training steps is number of batches * number of epochs.\n",
        "# total_steps = len(train_dataloader) * epochs\n",
        "\n",
        "# # Create the learning rate scheduler.\n",
        "# scheduler = get_linear_schedule_with_warmup(optimizer, \n",
        "#                                             num_warmup_steps = 0, # Default value in run_glue.py\n",
        "         \n",
        "\n",
        "# # This training code is based on the `run_glue.py` script here:\n",
        "# # https://github.com/huggingface/transformers/blob/5bfcd0485ece086ebcbed2d008813037968a9e58/examples/run_glue.py#L128\n",
        "\n",
        "\n",
        "# # Set the seed value all over the place to make this reproducible.\n",
        "# seed_val = 42\n",
        "\n",
        "# random.seed(seed_val)\n",
        "# np.random.seed(seed_val)\n",
        "# torch.manual_seed(seed_val)\n",
        "# torch.cuda.manual_seed_all(seed_val)\n",
        "\n",
        "# # Store the average loss after each epoch so we can plot them.\n",
        "# loss_values = []\n",
        "\n",
        "# # For each epoch...\n",
        "# for epoch_i in range(epochs):\n",
        "    \n",
        "#     # ========================================\n",
        "#     #               Training\n",
        "#     # ========================================\n",
        "    \n",
        "#     # Perform one full pass over the training set.\n",
        "\n",
        "#     print(\"\")\n",
        "#     print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))\n",
        "#     print('Training...')\n",
        "\n",
        "#     # Measure how long the training epoch takes.\n",
        "#     t0 = time.time()\n",
        "\n",
        "#     # Reset the total loss for this epoch.\n",
        "#     total_loss = 0\n",
        "\n",
        "#     # Put the model into training mode. Don't be mislead--the call to \n",
        "#     # `train` just changes the *mode*, it doesn't *perform* the training.\n",
        "#     # `dropout` and `batchnorm` layers behave differently during training\n",
        "#     # vs. test (source: https://stackoverflow.com/questions/51433378/what-does-model-train-do-in-pytorch)\n",
        "#     model.train()\n",
        "\n",
        "#     # For each batch of training data...\n",
        "#     for step, batch in enumerate(train_dataloader):\n",
        "\n",
        "#         # Progress update every 40 batches.\n",
        "#         if step % 40 == 0 and not step == 0:\n",
        "#             # Calculate elapsed time in minutes.\n",
        "#             elapsed = format_time(time.time() - t0)\n",
        "            \n",
        "#             # Report progress.\n",
        "#             print('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))\n",
        "\n",
        "#         # Unpack this training batch from our dataloader. \n",
        "#         #\n",
        "#         # As we unpack the batch, we'll also copy each tensor to the GPU using the \n",
        "#         # `to` method.\n",
        "#         #\n",
        "#         # `batch` contains three pytorch tensors:\n",
        "#         #   [0]: input ids \n",
        "#         #   [1]: attention masks\n",
        "#         #   [2]: labels \n",
        "#         b_input_ids = batch[0].to(device)\n",
        "#         b_input_mask = batch[1].to(device)\n",
        "#         b_labels = batch[2].to(device)\n",
        "\n",
        "#         # Always clear any previously calculated gradients before performing a\n",
        "#         # backward pass. PyTorch doesn't do this automatically because \n",
        "#         # accumulating the gradients is \"convenient while training RNNs\". \n",
        "#         # (source: https://stackoverflow.com/questions/48001598/why-do-we-need-to-call-zero-grad-in-pytorch)\n",
        "#         model.zero_grad()        \n",
        "\n",
        "#         # Perform a forward pass (evaluate the model on this training batch).\n",
        "#         # This will return the loss (rather than the model output) because we\n",
        "#         # have provided the `labels`.\n",
        "#         # The documentation for this `model` function is here: \n",
        "#         # https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#transformers.BertForSequenceClassification\n",
        "#         outputs = model(b_input_ids, \n",
        "#                     token_type_ids=None, \n",
        "#                     attention_mask=b_input_mask, \n",
        "#                     labels=b_labels)\n",
        "        \n",
        "#         # The call to `model` always returns a tuple, so we need to pull the \n",
        "#         # loss value out of the tuple.\n",
        "#         loss = outputs[0]\n",
        "\n",
        "#         # Accumulate the training loss over all of the batches so that we can\n",
        "#         # calculate the average loss at the end. `loss` is a Tensor containing a\n",
        "#         # single value; the `.item()` function just returns the Python value \n",
        "#         # from the tensor.\n",
        "#         total_loss += loss.item()\n",
        "\n",
        "#         # Perform a backward pass to calculate the gradients.\n",
        "#         loss.backward()\n",
        "\n",
        "#         # Clip the norm of the gradients to 1.0.\n",
        "#         # This is to help prevent the \"exploding gradients\" problem.\n",
        "#         torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
        "\n",
        "#         # Update parameters and take a step using the computed gradient.\n",
        "#         # The optimizer dictates the \"update rule\"--how the parameters are\n",
        "#         # modified based on their gradients, the learning rate, etc.\n",
        "#         optimizer.step()\n",
        "\n",
        "#         # Update the learning rate.\n",
        "#         scheduler.step()\n",
        "\n",
        "#     # Calculate the average loss over the training data.\n",
        "#     avg_train_loss = total_loss / len(train_dataloader)            \n",
        "    \n",
        "#     # Store the loss value for plotting the learning curve.\n",
        "#     loss_values.append(avg_train_loss)\n",
        "\n",
        "#     print(\"\")\n",
        "#     print(\"  Average training loss: {0:.2f}\".format(avg_train_loss))\n",
        "#     print(\"  Training epcoh took: {:}\".format(format_time(time.time() - t0)))\n",
        "        \n",
        "#     # ========================================\n",
        "#     #               Validation\n",
        "#     # ========================================\n",
        "#     # After the completion of each training epoch, measure our performance on\n",
        "#     # our validation set.\n",
        "\n",
        "#     print(\"\")\n",
        "#     print(\"Running Validation...\")\n",
        "\n",
        "#     t0 = time.time()\n",
        "\n",
        "#     # Put the model in evaluation mode--the dropout layers behave differently\n",
        "#     # during evaluation.\n",
        "#     model.eval()\n",
        "\n",
        "#     # Tracking variables \n",
        "#     eval_loss, eval_accuracy = 0, 0\n",
        "#     nb_eval_steps, nb_eval_examples = 0, 0\n",
        "\n",
        "#     # Evaluate data for one epoch\n",
        "#     for batch in validation_dataloader:\n",
        "        \n",
        "#         # Add batch to GPU\n",
        "#         batch = tuple(t.to(device) for t in batch)\n",
        "        \n",
        "#         # Unpack the inputs from our dataloader\n",
        "#         b_input_ids, b_input_mask, b_labels = batch\n",
        "        \n",
        "#         # Telling the model not to compute or store gradients, saving memory and\n",
        "#         # speeding up validation\n",
        "#         with torch.no_grad():        \n",
        "\n",
        "#             # Forward pass, calculate logit predictions.\n",
        "#             # This will return the logits rather than the loss because we have\n",
        "#             # not provided labels.\n",
        "#             # token_type_ids is the same as the \"segment ids\", which \n",
        "#             # differentiates sentence 1 and 2 in 2-sentence tasks.\n",
        "#             # The documentation for this `model` function is here: \n",
        "#             # https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#transformers.BertForSequenceClassification\n",
        "#             outputs = model(b_input_ids, \n",
        "#                             token_type_ids=None, \n",
        "#                             attention_mask=b_input_mask)\n",
        "        \n",
        "#         # Get the \"logits\" output by the model. The \"logits\" are the output\n",
        "#         # values prior to applying an activation function like the softmax.\n",
        "#         logits = outputs[0]\n",
        "\n",
        "#         # Move logits and labels to CPU\n",
        "#         logits = logits.detach().cpu().numpy()\n",
        "#         label_ids = b_labels.to('cpu').numpy()\n",
        "        \n",
        "#         # Calculate the accuracy for this batch of test sentences.\n",
        "#         tmp_eval_accuracy = flat_accuracy(logits, label_ids)\n",
        "        \n",
        "#         # Accumulate the total accuracy.\n",
        "#         eval_accuracy += tmp_eval_accuracy\n",
        "\n",
        "#         # Track the number of batches\n",
        "#         nb_eval_steps += 1\n",
        "\n",
        "#     # Report the final accuracy for this validation run.\n",
        "#     print(\"  Accuracy: {0:.2f}\".format(eval_accuracy/nb_eval_steps))\n",
        "#     print(\"  Validation took: {:}\".format(format_time(time.time() - t0)))\n",
        "\n",
        "# print(\"\")\n",
        "# print(\"Training complete!\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5UUm_34AgUeG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ex = examples[0]\n",
        "\n",
        "print(\"question : \", ex.question_text)\n",
        "print(\"text : \", ex.context_text)\n",
        "print(\"answer : \", ex.answer_text)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5sWdGrlp6YV0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#For single sequence input\n",
        "sentence = 'I really enjoyed this movie a lot.'\n",
        "tokens = tokenizer.tokenize(sentence)\n",
        "print(tokens)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "16UW4qrm2kga",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tokens = ['[CLS]'] + tokens + ['[SEP]']\n",
        "print(tokens)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yp227Kwq1wou",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "T = 12\n",
        "padded_tokens = tokens + ['[PAD]' for _ in range(T - len(tokens))]\n",
        "print(padded_tokens)\n",
        "# Out: ['[CLS]', 'i', 'really', 'enjoyed', 'this', 'movie', 'a', 'lot', '.', '[SEP]', '[PAD]', '[PAD]']\n",
        "attn_mask = [1 if token != '[PAD]' else 0 for token in padded_tokens]\n",
        "print(attn_mask)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4tIncLIZ2et3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
        "model = BertForQuestionAnswering.from_pretrained('bert-large-uncased-whole-word-masking-finetuned-squad')\n",
        "\n",
        "question, text = \"Who was Jim Henson?\", \"Jim Henson was a nice puppet\"\n",
        "input_ids = tokenizer.encode(question, text)\n",
        "print(input_ids)\n",
        "\n",
        "token_type_ids = [0 if i <= input_ids.index(102) else 1 for i in range(len(input_ids))]\n",
        "print(token_type_ids)\n",
        "\n",
        "start_scores, end_scores = model(torch.tensor([input_ids]), token_type_ids=torch.tensor([token_type_ids]))\n",
        "print(start_scores, end_scores)\n",
        "\n",
        "all_tokens = tokenizer.convert_ids_to_tokens(input_ids)\n",
        "answer = ' '.join(all_tokens[torch.argmax(start_scores) : torch.argmax(end_scores)+1])\n",
        "assert answer == \"a nice puppet\"\n",
        "print(answer)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Vi-U7eP475rh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}